crawler-user-agents 1.0.144 → 1.0.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci-validation.yml +4 -3
- package/crawler-user-agents.json +546 -941
- package/format.js +34 -0
- package/package.json +1 -1
package/crawler-user-agents.json
CHANGED
|
@@ -22,37 +22,32 @@
|
|
|
22
22
|
"Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
|
|
23
23
|
"SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
|
|
24
24
|
]
|
|
25
|
-
}
|
|
26
|
-
,
|
|
25
|
+
},
|
|
27
26
|
{
|
|
28
27
|
"pattern": "Googlebot-Image",
|
|
29
28
|
"instances": [
|
|
30
29
|
"Googlebot-Image/1.0"
|
|
31
30
|
]
|
|
32
|
-
}
|
|
33
|
-
,
|
|
31
|
+
},
|
|
34
32
|
{
|
|
35
33
|
"pattern": "Googlebot-News",
|
|
36
34
|
"instances": [
|
|
37
35
|
"Googlebot-News"
|
|
38
36
|
]
|
|
39
|
-
}
|
|
40
|
-
,
|
|
37
|
+
},
|
|
41
38
|
{
|
|
42
39
|
"pattern": "Googlebot-Video",
|
|
43
40
|
"instances": [
|
|
44
41
|
"Googlebot-Video/1.0"
|
|
45
42
|
]
|
|
46
|
-
}
|
|
47
|
-
,
|
|
43
|
+
},
|
|
48
44
|
{
|
|
49
45
|
"pattern": "AdsBot-Google([^-]|$)",
|
|
50
46
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
51
47
|
"instances": [
|
|
52
48
|
"AdsBot-Google (+http://www.google.com/adsbot.html)"
|
|
53
49
|
]
|
|
54
|
-
}
|
|
55
|
-
,
|
|
50
|
+
},
|
|
56
51
|
{
|
|
57
52
|
"pattern": "AdsBot-Google-Mobile",
|
|
58
53
|
"addition_date": "2017/08/21",
|
|
@@ -62,8 +57,7 @@
|
|
|
62
57
|
"Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
|
|
63
58
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"
|
|
64
59
|
]
|
|
65
|
-
}
|
|
66
|
-
,
|
|
60
|
+
},
|
|
67
61
|
{
|
|
68
62
|
"pattern": "Feedfetcher-Google",
|
|
69
63
|
"addition_date": "2018/06/27",
|
|
@@ -71,8 +65,7 @@
|
|
|
71
65
|
"instances": [
|
|
72
66
|
"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)"
|
|
73
67
|
]
|
|
74
|
-
}
|
|
75
|
-
,
|
|
68
|
+
},
|
|
76
69
|
{
|
|
77
70
|
"pattern": "Mediapartners-Google",
|
|
78
71
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
@@ -82,15 +75,13 @@
|
|
|
82
75
|
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
|
|
83
76
|
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
|
|
84
77
|
]
|
|
85
|
-
}
|
|
86
|
-
,
|
|
78
|
+
},
|
|
87
79
|
{
|
|
88
80
|
"pattern": "Mediapartners \\(Googlebot\\)",
|
|
89
81
|
"addition_date": "2017/08/08",
|
|
90
82
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
91
83
|
"instances": []
|
|
92
|
-
}
|
|
93
|
-
,
|
|
84
|
+
},
|
|
94
85
|
{
|
|
95
86
|
"pattern": "APIs-Google",
|
|
96
87
|
"addition_date": "2017/08/08",
|
|
@@ -98,8 +89,7 @@
|
|
|
98
89
|
"instances": [
|
|
99
90
|
"APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"
|
|
100
91
|
]
|
|
101
|
-
}
|
|
102
|
-
,
|
|
92
|
+
},
|
|
103
93
|
{
|
|
104
94
|
"pattern": "Google-InspectionTool",
|
|
105
95
|
"url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
|
|
@@ -107,8 +97,7 @@
|
|
|
107
97
|
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)",
|
|
108
98
|
"Mozilla/5.0 (compatible; Google-InspectionTool/1.0)"
|
|
109
99
|
]
|
|
110
|
-
}
|
|
111
|
-
,
|
|
100
|
+
},
|
|
112
101
|
{
|
|
113
102
|
"pattern": "Storebot-Google",
|
|
114
103
|
"url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
|
|
@@ -116,16 +105,14 @@
|
|
|
116
105
|
"Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
|
117
106
|
"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"
|
|
118
107
|
]
|
|
119
|
-
}
|
|
120
|
-
,
|
|
108
|
+
},
|
|
121
109
|
{
|
|
122
110
|
"pattern": "GoogleOther",
|
|
123
111
|
"url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
|
|
124
112
|
"instances": [
|
|
125
113
|
"GoogleOther"
|
|
126
114
|
]
|
|
127
|
-
}
|
|
128
|
-
,
|
|
115
|
+
},
|
|
129
116
|
{
|
|
130
117
|
"pattern": "bingbot",
|
|
131
118
|
"url": "http://www.bing.com/bingbot.htm",
|
|
@@ -145,8 +132,7 @@
|
|
|
145
132
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36",
|
|
146
133
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36"
|
|
147
134
|
]
|
|
148
|
-
}
|
|
149
|
-
,
|
|
135
|
+
},
|
|
150
136
|
{
|
|
151
137
|
"pattern": "Slurp",
|
|
152
138
|
"url": "http://help.yahoo.com/help/us/ysearch/slurp",
|
|
@@ -155,8 +141,7 @@
|
|
|
155
141
|
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
|
|
156
142
|
"Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
|
157
143
|
]
|
|
158
|
-
}
|
|
159
|
-
,
|
|
144
|
+
},
|
|
160
145
|
{
|
|
161
146
|
"pattern": "[wW]get",
|
|
162
147
|
"instances": [
|
|
@@ -164,8 +149,7 @@
|
|
|
164
149
|
"Wget/1.14 (linux-gnu)",
|
|
165
150
|
"Wget/1.20.3 (linux-gnu)"
|
|
166
151
|
]
|
|
167
|
-
}
|
|
168
|
-
,
|
|
152
|
+
},
|
|
169
153
|
{
|
|
170
154
|
"pattern": "LinkedInBot",
|
|
171
155
|
"instances": [
|
|
@@ -173,8 +157,7 @@
|
|
|
173
157
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)",
|
|
174
158
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)"
|
|
175
159
|
]
|
|
176
|
-
}
|
|
177
|
-
,
|
|
160
|
+
},
|
|
178
161
|
{
|
|
179
162
|
"pattern": "Python-urllib",
|
|
180
163
|
"instances": [
|
|
@@ -189,9 +172,8 @@
|
|
|
189
172
|
"Python-urllib/3.5",
|
|
190
173
|
"Python-urllib/3.6",
|
|
191
174
|
"Python-urllib/3.7"
|
|
192
|
-
|
|
193
|
-
}
|
|
194
|
-
,
|
|
175
|
+
]
|
|
176
|
+
},
|
|
195
177
|
{
|
|
196
178
|
"pattern": "python-requests",
|
|
197
179
|
"addition_date": "2018/05/27",
|
|
@@ -204,8 +186,7 @@
|
|
|
204
186
|
"python-requests/2.21.0",
|
|
205
187
|
"python-requests/2.22.0"
|
|
206
188
|
]
|
|
207
|
-
}
|
|
208
|
-
,
|
|
189
|
+
},
|
|
209
190
|
{
|
|
210
191
|
"pattern": "aiohttp",
|
|
211
192
|
"addition_date": "2019/12/23",
|
|
@@ -215,19 +196,16 @@
|
|
|
215
196
|
"Python/3.7 aiohttp/3.6.2a2"
|
|
216
197
|
],
|
|
217
198
|
"url": "https://docs.aiohttp.org/en/stable/"
|
|
218
|
-
}
|
|
219
|
-
,
|
|
199
|
+
},
|
|
220
200
|
{
|
|
221
201
|
"pattern": "httpx",
|
|
222
202
|
"addition_date": "2019/12/23",
|
|
223
203
|
"instances": [
|
|
224
204
|
"python-httpx/0.16.1",
|
|
225
205
|
"python-httpx/0.13.0.dev1"
|
|
226
|
-
|
|
227
206
|
],
|
|
228
207
|
"url": "https://www.python-httpx.org"
|
|
229
|
-
}
|
|
230
|
-
,
|
|
208
|
+
},
|
|
231
209
|
{
|
|
232
210
|
"pattern": "libwww-perl",
|
|
233
211
|
"instances": [
|
|
@@ -235,15 +213,13 @@
|
|
|
235
213
|
"2Bone_LinkChkr/1.0 libwww-perl/6.03",
|
|
236
214
|
"amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"
|
|
237
215
|
]
|
|
238
|
-
}
|
|
239
|
-
,
|
|
216
|
+
},
|
|
240
217
|
{
|
|
241
218
|
"pattern": "httpunit",
|
|
242
219
|
"instances": [
|
|
243
220
|
"httpunit/1.x"
|
|
244
221
|
]
|
|
245
|
-
}
|
|
246
|
-
,
|
|
222
|
+
},
|
|
247
223
|
{
|
|
248
224
|
"pattern": "Nutch",
|
|
249
225
|
"instances": [
|
|
@@ -251,8 +227,7 @@
|
|
|
251
227
|
"NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
|
|
252
228
|
"istellabot-nutch/Nutch-1.10"
|
|
253
229
|
]
|
|
254
|
-
}
|
|
255
|
-
,
|
|
230
|
+
},
|
|
256
231
|
{
|
|
257
232
|
"pattern": "Go-http-client",
|
|
258
233
|
"addition_date": "2016/03/26",
|
|
@@ -261,8 +236,7 @@
|
|
|
261
236
|
"Go-http-client/1.1",
|
|
262
237
|
"Go-http-client/2.0"
|
|
263
238
|
]
|
|
264
|
-
}
|
|
265
|
-
,
|
|
239
|
+
},
|
|
266
240
|
{
|
|
267
241
|
"pattern": "phpcrawl",
|
|
268
242
|
"addition_date": "2012/09/17",
|
|
@@ -270,8 +244,7 @@
|
|
|
270
244
|
"instances": [
|
|
271
245
|
"phpcrawl"
|
|
272
246
|
]
|
|
273
|
-
}
|
|
274
|
-
,
|
|
247
|
+
},
|
|
275
248
|
{
|
|
276
249
|
"pattern": "msnbot",
|
|
277
250
|
"url": "http://search.msn.com/msnbot.htm",
|
|
@@ -291,13 +264,11 @@
|
|
|
291
264
|
"msnbot/2.0b (+http://search.msn.com/msnbot.htm).",
|
|
292
265
|
"msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"
|
|
293
266
|
]
|
|
294
|
-
}
|
|
295
|
-
,
|
|
267
|
+
},
|
|
296
268
|
{
|
|
297
269
|
"pattern": "jyxobot",
|
|
298
270
|
"instances": []
|
|
299
|
-
}
|
|
300
|
-
,
|
|
271
|
+
},
|
|
301
272
|
{
|
|
302
273
|
"pattern": "FAST-WebCrawler",
|
|
303
274
|
"instances": [
|
|
@@ -306,23 +277,20 @@
|
|
|
306
277
|
"FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
|
|
307
278
|
"FAST-WebCrawler/3.8"
|
|
308
279
|
]
|
|
309
|
-
}
|
|
310
|
-
,
|
|
280
|
+
},
|
|
311
281
|
{
|
|
312
282
|
"pattern": "FAST Enterprise Crawler",
|
|
313
283
|
"instances": [
|
|
314
284
|
"FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
|
|
315
285
|
"FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"
|
|
316
286
|
]
|
|
317
|
-
}
|
|
318
|
-
,
|
|
287
|
+
},
|
|
319
288
|
{
|
|
320
289
|
"pattern": "BIGLOTRON",
|
|
321
290
|
"instances": [
|
|
322
291
|
"BIGLOTRON (Beta 2;GNU/Linux)"
|
|
323
292
|
]
|
|
324
|
-
}
|
|
325
|
-
,
|
|
293
|
+
},
|
|
326
294
|
{
|
|
327
295
|
"pattern": "Teoma",
|
|
328
296
|
"instances": [
|
|
@@ -330,24 +298,21 @@
|
|
|
330
298
|
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"
|
|
331
299
|
],
|
|
332
300
|
"url": "http://about.ask.com/en/docs/about/webmasters.shtml"
|
|
333
|
-
}
|
|
334
|
-
,
|
|
301
|
+
},
|
|
335
302
|
{
|
|
336
303
|
"pattern": "convera",
|
|
337
304
|
"instances": [
|
|
338
305
|
"ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"
|
|
339
306
|
],
|
|
340
307
|
"url": "http://ews.converasearch.com/crawl.htm"
|
|
341
|
-
}
|
|
342
|
-
,
|
|
308
|
+
},
|
|
343
309
|
{
|
|
344
310
|
"pattern": "seekbot",
|
|
345
311
|
"instances": [
|
|
346
312
|
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"
|
|
347
313
|
],
|
|
348
314
|
"url": "http://www.seekbot.net/bot.html"
|
|
349
|
-
}
|
|
350
|
-
,
|
|
315
|
+
},
|
|
351
316
|
{
|
|
352
317
|
"pattern": "Gigabot",
|
|
353
318
|
"instances": [
|
|
@@ -355,16 +320,14 @@
|
|
|
355
320
|
"Gigabot/2.0 (http://www.gigablast.com/spider.html)"
|
|
356
321
|
],
|
|
357
322
|
"url": "http://www.gigablast.com/spider.html"
|
|
358
|
-
}
|
|
359
|
-
,
|
|
323
|
+
},
|
|
360
324
|
{
|
|
361
325
|
"pattern": "Gigablast",
|
|
362
326
|
"instances": [
|
|
363
327
|
"GigablastOpenSource/1.0"
|
|
364
328
|
],
|
|
365
329
|
"url": "https://github.com/gigablast/open-source-search-engine"
|
|
366
|
-
}
|
|
367
|
-
,
|
|
330
|
+
},
|
|
368
331
|
{
|
|
369
332
|
"pattern": "exabot",
|
|
370
333
|
"instances": [
|
|
@@ -375,35 +338,30 @@
|
|
|
375
338
|
"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",
|
|
376
339
|
"Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)"
|
|
377
340
|
]
|
|
378
|
-
}
|
|
379
|
-
,
|
|
341
|
+
},
|
|
380
342
|
{
|
|
381
343
|
"pattern": "ia_archiver",
|
|
382
344
|
"instances": [
|
|
383
345
|
"ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
|
|
384
346
|
"ia_archiver-web.archive.org"
|
|
385
347
|
]
|
|
386
|
-
}
|
|
387
|
-
,
|
|
348
|
+
},
|
|
388
349
|
{
|
|
389
350
|
"pattern": "GingerCrawler",
|
|
390
351
|
"instances": [
|
|
391
352
|
"GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"
|
|
392
353
|
]
|
|
393
|
-
}
|
|
394
|
-
,
|
|
354
|
+
},
|
|
395
355
|
{
|
|
396
356
|
"pattern": "webmon ",
|
|
397
357
|
"instances": []
|
|
398
|
-
}
|
|
399
|
-
,
|
|
358
|
+
},
|
|
400
359
|
{
|
|
401
360
|
"pattern": "HTTrack",
|
|
402
361
|
"instances": [
|
|
403
362
|
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"
|
|
404
363
|
]
|
|
405
|
-
}
|
|
406
|
-
,
|
|
364
|
+
},
|
|
407
365
|
{
|
|
408
366
|
"pattern": "grub\\.org",
|
|
409
367
|
"instances": [
|
|
@@ -419,23 +377,19 @@
|
|
|
419
377
|
"Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",
|
|
420
378
|
"Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"
|
|
421
379
|
]
|
|
422
|
-
}
|
|
423
|
-
,
|
|
380
|
+
},
|
|
424
381
|
{
|
|
425
382
|
"pattern": "UsineNouvelleCrawler",
|
|
426
383
|
"instances": []
|
|
427
|
-
}
|
|
428
|
-
,
|
|
384
|
+
},
|
|
429
385
|
{
|
|
430
386
|
"pattern": "antibot",
|
|
431
387
|
"instances": []
|
|
432
|
-
}
|
|
433
|
-
,
|
|
388
|
+
},
|
|
434
389
|
{
|
|
435
390
|
"pattern": "netresearchserver",
|
|
436
391
|
"instances": []
|
|
437
|
-
}
|
|
438
|
-
,
|
|
392
|
+
},
|
|
439
393
|
{
|
|
440
394
|
"pattern": "speedy",
|
|
441
395
|
"instances": [
|
|
@@ -445,13 +399,11 @@
|
|
|
445
399
|
"Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",
|
|
446
400
|
"Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"
|
|
447
401
|
]
|
|
448
|
-
}
|
|
449
|
-
,
|
|
402
|
+
},
|
|
450
403
|
{
|
|
451
404
|
"pattern": "fluffy",
|
|
452
405
|
"instances": []
|
|
453
|
-
}
|
|
454
|
-
,
|
|
406
|
+
},
|
|
455
407
|
{
|
|
456
408
|
"pattern": "findlink",
|
|
457
409
|
"instances": [
|
|
@@ -479,20 +431,17 @@
|
|
|
479
431
|
"findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
|
|
480
432
|
"findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"
|
|
481
433
|
]
|
|
482
|
-
}
|
|
483
|
-
,
|
|
434
|
+
},
|
|
484
435
|
{
|
|
485
436
|
"pattern": "msrbot",
|
|
486
437
|
"instances": []
|
|
487
|
-
}
|
|
488
|
-
,
|
|
438
|
+
},
|
|
489
439
|
{
|
|
490
440
|
"pattern": "panscient",
|
|
491
441
|
"instances": [
|
|
492
442
|
"panscient.com"
|
|
493
443
|
]
|
|
494
|
-
}
|
|
495
|
-
,
|
|
444
|
+
},
|
|
496
445
|
{
|
|
497
446
|
"pattern": "yacybot",
|
|
498
447
|
"instances": [
|
|
@@ -544,13 +493,11 @@
|
|
|
544
493
|
"yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html",
|
|
545
494
|
"yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html"
|
|
546
495
|
]
|
|
547
|
-
}
|
|
548
|
-
,
|
|
496
|
+
},
|
|
549
497
|
{
|
|
550
498
|
"pattern": "AISearchBot",
|
|
551
499
|
"instances": []
|
|
552
|
-
}
|
|
553
|
-
,
|
|
500
|
+
},
|
|
554
501
|
{
|
|
555
502
|
"pattern": "ips-agent",
|
|
556
503
|
"instances": [
|
|
@@ -560,13 +507,11 @@
|
|
|
560
507
|
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",
|
|
561
508
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"
|
|
562
509
|
]
|
|
563
|
-
}
|
|
564
|
-
,
|
|
510
|
+
},
|
|
565
511
|
{
|
|
566
512
|
"pattern": "tagoobot",
|
|
567
513
|
"instances": []
|
|
568
|
-
}
|
|
569
|
-
,
|
|
514
|
+
},
|
|
570
515
|
{
|
|
571
516
|
"pattern": "MJ12bot",
|
|
572
517
|
"instances": [
|
|
@@ -591,37 +536,32 @@
|
|
|
591
536
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)",
|
|
592
537
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"
|
|
593
538
|
]
|
|
594
|
-
}
|
|
595
|
-
,
|
|
539
|
+
},
|
|
596
540
|
{
|
|
597
541
|
"pattern": "woriobot",
|
|
598
542
|
"instances": [
|
|
599
543
|
"Mozilla/5.0 (compatible; woriobot +http://worio.com)",
|
|
600
544
|
"Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"
|
|
601
545
|
]
|
|
602
|
-
}
|
|
603
|
-
,
|
|
546
|
+
},
|
|
604
547
|
{
|
|
605
548
|
"pattern": "yanga",
|
|
606
549
|
"instances": [
|
|
607
550
|
"Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"
|
|
608
551
|
]
|
|
609
|
-
}
|
|
610
|
-
,
|
|
552
|
+
},
|
|
611
553
|
{
|
|
612
554
|
"pattern": "buzzbot",
|
|
613
555
|
"instances": [
|
|
614
556
|
"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"
|
|
615
557
|
]
|
|
616
|
-
}
|
|
617
|
-
,
|
|
558
|
+
},
|
|
618
559
|
{
|
|
619
560
|
"pattern": "mlbot",
|
|
620
561
|
"instances": [
|
|
621
562
|
"MLBot (www.metadatalabs.com/mlbot)"
|
|
622
563
|
]
|
|
623
|
-
}
|
|
624
|
-
,
|
|
564
|
+
},
|
|
625
565
|
{
|
|
626
566
|
"pattern": "yandex\\.com\\/bots",
|
|
627
567
|
"url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs",
|
|
@@ -669,14 +609,12 @@
|
|
|
669
609
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)"
|
|
670
610
|
],
|
|
671
611
|
"addition_date": "2015/04/14"
|
|
672
|
-
}
|
|
673
|
-
,
|
|
612
|
+
},
|
|
674
613
|
{
|
|
675
614
|
"pattern": "purebot",
|
|
676
615
|
"addition_date": "2010/01/19",
|
|
677
616
|
"instances": []
|
|
678
|
-
}
|
|
679
|
-
,
|
|
617
|
+
},
|
|
680
618
|
{
|
|
681
619
|
"pattern": "Linguee Bot",
|
|
682
620
|
"addition_date": "2010/01/26",
|
|
@@ -685,8 +623,7 @@
|
|
|
685
623
|
"Linguee Bot (http://www.linguee.com/bot)",
|
|
686
624
|
"Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"
|
|
687
625
|
]
|
|
688
|
-
}
|
|
689
|
-
,
|
|
626
|
+
},
|
|
690
627
|
{
|
|
691
628
|
"pattern": "CyberPatrol",
|
|
692
629
|
"addition_date": "2010/02/11",
|
|
@@ -694,8 +631,7 @@
|
|
|
694
631
|
"instances": [
|
|
695
632
|
"CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"
|
|
696
633
|
]
|
|
697
|
-
}
|
|
698
|
-
,
|
|
634
|
+
},
|
|
699
635
|
{
|
|
700
636
|
"pattern": "voilabot",
|
|
701
637
|
"addition_date": "2010/05/18",
|
|
@@ -703,8 +639,7 @@
|
|
|
703
639
|
"Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
|
|
704
640
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)"
|
|
705
641
|
]
|
|
706
|
-
}
|
|
707
|
-
,
|
|
642
|
+
},
|
|
708
643
|
{
|
|
709
644
|
"pattern": "Baiduspider",
|
|
710
645
|
"addition_date": "2010/07/15",
|
|
@@ -713,14 +648,12 @@
|
|
|
713
648
|
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
|
|
714
649
|
"Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)"
|
|
715
650
|
]
|
|
716
|
-
}
|
|
717
|
-
,
|
|
651
|
+
},
|
|
718
652
|
{
|
|
719
653
|
"pattern": "citeseerxbot",
|
|
720
654
|
"addition_date": "2010/07/17",
|
|
721
655
|
"instances": []
|
|
722
|
-
}
|
|
723
|
-
,
|
|
656
|
+
},
|
|
724
657
|
{
|
|
725
658
|
"pattern": "spbot",
|
|
726
659
|
"addition_date": "2010/07/31",
|
|
@@ -761,15 +694,13 @@
|
|
|
761
694
|
"Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",
|
|
762
695
|
"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"
|
|
763
696
|
]
|
|
764
|
-
}
|
|
765
|
-
,
|
|
697
|
+
},
|
|
766
698
|
{
|
|
767
699
|
"pattern": "twengabot",
|
|
768
700
|
"addition_date": "2010/08/03",
|
|
769
701
|
"url": "http://www.twenga.com/bot.html",
|
|
770
702
|
"instances": []
|
|
771
|
-
}
|
|
772
|
-
,
|
|
703
|
+
},
|
|
773
704
|
{
|
|
774
705
|
"pattern": "postrank",
|
|
775
706
|
"addition_date": "2010/08/03",
|
|
@@ -778,8 +709,7 @@
|
|
|
778
709
|
"PostRank/2.0 (postrank.com)",
|
|
779
710
|
"PostRank/2.0 (postrank.com; 1 subscribers)"
|
|
780
711
|
]
|
|
781
|
-
}
|
|
782
|
-
,
|
|
712
|
+
},
|
|
783
713
|
{
|
|
784
714
|
"pattern": "Turnitin",
|
|
785
715
|
"addition_date": "2010/09/26",
|
|
@@ -788,15 +718,13 @@
|
|
|
788
718
|
"TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)",
|
|
789
719
|
"Turnitin (https://bit.ly/2UvnfoQ)"
|
|
790
720
|
]
|
|
791
|
-
}
|
|
792
|
-
,
|
|
721
|
+
},
|
|
793
722
|
{
|
|
794
723
|
"pattern": "scribdbot",
|
|
795
724
|
"addition_date": "2010/09/28",
|
|
796
725
|
"url": "http://www.scribd.com",
|
|
797
726
|
"instances": []
|
|
798
|
-
}
|
|
799
|
-
,
|
|
727
|
+
},
|
|
800
728
|
{
|
|
801
729
|
"pattern": "page2rss",
|
|
802
730
|
"addition_date": "2010/10/07",
|
|
@@ -804,8 +732,7 @@
|
|
|
804
732
|
"instances": [
|
|
805
733
|
"Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)"
|
|
806
734
|
]
|
|
807
|
-
}
|
|
808
|
-
,
|
|
735
|
+
},
|
|
809
736
|
{
|
|
810
737
|
"pattern": "sitebot",
|
|
811
738
|
"addition_date": "2010/12/15",
|
|
@@ -813,8 +740,7 @@
|
|
|
813
740
|
"instances": [
|
|
814
741
|
"Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"
|
|
815
742
|
]
|
|
816
|
-
}
|
|
817
|
-
,
|
|
743
|
+
},
|
|
818
744
|
{
|
|
819
745
|
"pattern": "linkdex",
|
|
820
746
|
"addition_date": "2011/01/06",
|
|
@@ -828,14 +754,12 @@
|
|
|
828
754
|
"linkdex.com/v2.0",
|
|
829
755
|
"linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"
|
|
830
756
|
]
|
|
831
|
-
}
|
|
832
|
-
,
|
|
757
|
+
},
|
|
833
758
|
{
|
|
834
759
|
"pattern": "Adidxbot",
|
|
835
760
|
"url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",
|
|
836
761
|
"instances": []
|
|
837
|
-
}
|
|
838
|
-
,
|
|
762
|
+
},
|
|
839
763
|
{
|
|
840
764
|
"pattern": "ezooms",
|
|
841
765
|
"addition_date": "2011/04/27",
|
|
@@ -843,8 +767,7 @@
|
|
|
843
767
|
"instances": [
|
|
844
768
|
"Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"
|
|
845
769
|
]
|
|
846
|
-
}
|
|
847
|
-
,
|
|
770
|
+
},
|
|
848
771
|
{
|
|
849
772
|
"pattern": "dotbot",
|
|
850
773
|
"addition_date": "2011/04/27",
|
|
@@ -852,8 +775,7 @@
|
|
|
852
775
|
"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",
|
|
853
776
|
"dotbot"
|
|
854
777
|
]
|
|
855
|
-
}
|
|
856
|
-
,
|
|
778
|
+
},
|
|
857
779
|
{
|
|
858
780
|
"pattern": "Mail\\.RU_Bot",
|
|
859
781
|
"addition_date": "2011/04/27",
|
|
@@ -863,8 +785,7 @@
|
|
|
863
785
|
"Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/",
|
|
864
786
|
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)"
|
|
865
787
|
]
|
|
866
|
-
}
|
|
867
|
-
,
|
|
788
|
+
},
|
|
868
789
|
{
|
|
869
790
|
"pattern": "discobot",
|
|
870
791
|
"addition_date": "2011/05/03",
|
|
@@ -874,8 +795,7 @@
|
|
|
874
795
|
"Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",
|
|
875
796
|
"mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"
|
|
876
797
|
]
|
|
877
|
-
}
|
|
878
|
-
,
|
|
798
|
+
},
|
|
879
799
|
{
|
|
880
800
|
"pattern": "heritrix",
|
|
881
801
|
"addition_date": "2011/06/21",
|
|
@@ -904,15 +824,13 @@
|
|
|
904
824
|
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
|
|
905
825
|
"Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"
|
|
906
826
|
]
|
|
907
|
-
}
|
|
908
|
-
,
|
|
827
|
+
},
|
|
909
828
|
{
|
|
910
829
|
"pattern": "findthatfile",
|
|
911
830
|
"addition_date": "2011/06/21",
|
|
912
831
|
"url": "http://www.findthatfile.com/",
|
|
913
832
|
"instances": []
|
|
914
|
-
}
|
|
915
|
-
,
|
|
833
|
+
},
|
|
916
834
|
{
|
|
917
835
|
"pattern": "europarchive\\.org",
|
|
918
836
|
"addition_date": "2011/06/21",
|
|
@@ -920,8 +838,7 @@
|
|
|
920
838
|
"instances": [
|
|
921
839
|
"Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"
|
|
922
840
|
]
|
|
923
|
-
}
|
|
924
|
-
,
|
|
841
|
+
},
|
|
925
842
|
{
|
|
926
843
|
"pattern": "NerdByNature\\.Bot",
|
|
927
844
|
"addition_date": "2011/07/12",
|
|
@@ -929,14 +846,12 @@
|
|
|
929
846
|
"instances": [
|
|
930
847
|
"Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"
|
|
931
848
|
]
|
|
932
|
-
}
|
|
933
|
-
,
|
|
849
|
+
},
|
|
934
850
|
{
|
|
935
851
|
"pattern": "sistrix crawler",
|
|
936
852
|
"addition_date": "2011/08/02",
|
|
937
853
|
"instances": []
|
|
938
|
-
}
|
|
939
|
-
,
|
|
854
|
+
},
|
|
940
855
|
{
|
|
941
856
|
"pattern": "Ahrefs(Bot|SiteAudit)",
|
|
942
857
|
"addition_date": "2011/08/28",
|
|
@@ -949,104 +864,91 @@
|
|
|
949
864
|
"Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)",
|
|
950
865
|
"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"
|
|
951
866
|
]
|
|
952
|
-
}
|
|
953
|
-
,
|
|
867
|
+
},
|
|
954
868
|
{
|
|
955
869
|
"pattern": "fuelbot",
|
|
956
870
|
"addition_date": "2018/06/28",
|
|
957
871
|
"instances": [
|
|
958
872
|
"fuelbot"
|
|
959
873
|
]
|
|
960
|
-
}
|
|
961
|
-
,
|
|
874
|
+
},
|
|
962
875
|
{
|
|
963
876
|
"pattern": "CrunchBot",
|
|
964
877
|
"addition_date": "2018/06/28",
|
|
965
878
|
"instances": [
|
|
966
879
|
"CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)"
|
|
967
880
|
]
|
|
968
|
-
}
|
|
969
|
-
,
|
|
881
|
+
},
|
|
970
882
|
{
|
|
971
883
|
"pattern": "IndeedBot",
|
|
972
884
|
"addition_date": "2018/06/28",
|
|
973
885
|
"instances": [
|
|
974
886
|
"Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)"
|
|
975
887
|
]
|
|
976
|
-
}
|
|
977
|
-
,
|
|
888
|
+
},
|
|
978
889
|
{
|
|
979
890
|
"pattern": "mappydata",
|
|
980
891
|
"addition_date": "2018/06/28",
|
|
981
892
|
"instances": [
|
|
982
893
|
"Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)"
|
|
983
894
|
]
|
|
984
|
-
}
|
|
985
|
-
,
|
|
895
|
+
},
|
|
986
896
|
{
|
|
987
897
|
"pattern": "woobot",
|
|
988
898
|
"addition_date": "2018/06/28",
|
|
989
899
|
"instances": [
|
|
990
900
|
"woobot"
|
|
991
901
|
]
|
|
992
|
-
}
|
|
993
|
-
,
|
|
902
|
+
},
|
|
994
903
|
{
|
|
995
904
|
"pattern": "ZoominfoBot",
|
|
996
905
|
"addition_date": "2018/06/28",
|
|
997
906
|
"instances": [
|
|
998
907
|
"ZoominfoBot (zoominfobot at zoominfo dot com)"
|
|
999
908
|
]
|
|
1000
|
-
}
|
|
1001
|
-
,
|
|
909
|
+
},
|
|
1002
910
|
{
|
|
1003
911
|
"pattern": "PrivacyAwareBot",
|
|
1004
912
|
"addition_date": "2018/06/28",
|
|
1005
913
|
"instances": [
|
|
1006
914
|
"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)"
|
|
1007
915
|
]
|
|
1008
|
-
}
|
|
1009
|
-
,
|
|
916
|
+
},
|
|
1010
917
|
{
|
|
1011
918
|
"pattern": "Multiviewbot",
|
|
1012
919
|
"addition_date": "2018/06/28",
|
|
1013
920
|
"instances": [
|
|
1014
921
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot"
|
|
1015
922
|
]
|
|
1016
|
-
}
|
|
1017
|
-
,
|
|
923
|
+
},
|
|
1018
924
|
{
|
|
1019
925
|
"pattern": "SWIMGBot",
|
|
1020
926
|
"addition_date": "2018/06/28",
|
|
1021
927
|
"instances": [
|
|
1022
928
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot"
|
|
1023
929
|
]
|
|
1024
|
-
}
|
|
1025
|
-
,
|
|
930
|
+
},
|
|
1026
931
|
{
|
|
1027
932
|
"pattern": "Grobbot",
|
|
1028
933
|
"addition_date": "2018/06/28",
|
|
1029
934
|
"instances": [
|
|
1030
935
|
"Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)"
|
|
1031
936
|
]
|
|
1032
|
-
}
|
|
1033
|
-
,
|
|
937
|
+
},
|
|
1034
938
|
{
|
|
1035
939
|
"pattern": "eright",
|
|
1036
940
|
"addition_date": "2018/06/28",
|
|
1037
941
|
"instances": [
|
|
1038
942
|
"Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)"
|
|
1039
943
|
]
|
|
1040
|
-
}
|
|
1041
|
-
,
|
|
944
|
+
},
|
|
1042
945
|
{
|
|
1043
946
|
"pattern": "Apercite",
|
|
1044
947
|
"addition_date": "2018/06/28",
|
|
1045
948
|
"instances": [
|
|
1046
949
|
"Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)"
|
|
1047
950
|
]
|
|
1048
|
-
}
|
|
1049
|
-
,
|
|
951
|
+
},
|
|
1050
952
|
{
|
|
1051
953
|
"pattern": "semanticbot",
|
|
1052
954
|
"addition_date": "2018/06/28",
|
|
@@ -1054,8 +956,7 @@
|
|
|
1054
956
|
"semanticbot",
|
|
1055
957
|
"semanticbot (info@semanticaudience.com)"
|
|
1056
958
|
]
|
|
1057
|
-
}
|
|
1058
|
-
,
|
|
959
|
+
},
|
|
1059
960
|
{
|
|
1060
961
|
"pattern": "Aboundex",
|
|
1061
962
|
"addition_date": "2011/09/28",
|
|
@@ -1064,23 +965,20 @@
|
|
|
1064
965
|
"Aboundex/0.2 (http://www.aboundex.com/crawler/)",
|
|
1065
966
|
"Aboundex/0.3 (http://www.aboundex.com/crawler/)"
|
|
1066
967
|
]
|
|
1067
|
-
}
|
|
1068
|
-
,
|
|
968
|
+
},
|
|
1069
969
|
{
|
|
1070
970
|
"pattern": "domaincrawler",
|
|
1071
971
|
"addition_date": "2011/10/21",
|
|
1072
972
|
"instances": [
|
|
1073
973
|
"CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"
|
|
1074
974
|
]
|
|
1075
|
-
}
|
|
1076
|
-
,
|
|
975
|
+
},
|
|
1077
976
|
{
|
|
1078
977
|
"pattern": "wbsearchbot",
|
|
1079
978
|
"addition_date": "2011/12/21",
|
|
1080
979
|
"url": "http://www.warebay.com/bot.html",
|
|
1081
980
|
"instances": []
|
|
1082
|
-
}
|
|
1083
|
-
,
|
|
981
|
+
},
|
|
1084
982
|
{
|
|
1085
983
|
"pattern": "summify",
|
|
1086
984
|
"addition_date": "2012/01/04",
|
|
@@ -1088,8 +986,7 @@
|
|
|
1088
986
|
"instances": [
|
|
1089
987
|
"Summify (Summify/1.0.1; +http://summify.com)"
|
|
1090
988
|
]
|
|
1091
|
-
}
|
|
1092
|
-
,
|
|
989
|
+
},
|
|
1093
990
|
{
|
|
1094
991
|
"pattern": "CCBot",
|
|
1095
992
|
"addition_date": "2012/02/05",
|
|
@@ -1098,14 +995,12 @@
|
|
|
1098
995
|
"CCBot/2.0 (http://commoncrawl.org/faq/)",
|
|
1099
996
|
"CCBot/2.0 (https://commoncrawl.org/faq/)"
|
|
1100
997
|
]
|
|
1101
|
-
}
|
|
1102
|
-
,
|
|
998
|
+
},
|
|
1103
999
|
{
|
|
1104
1000
|
"pattern": "edisterbot",
|
|
1105
1001
|
"addition_date": "2012/02/25",
|
|
1106
1002
|
"instances": []
|
|
1107
|
-
}
|
|
1108
|
-
,
|
|
1003
|
+
},
|
|
1109
1004
|
{
|
|
1110
1005
|
"pattern": "SeznamBot",
|
|
1111
1006
|
"addition_date": "2012/03/14",
|
|
@@ -1117,36 +1012,31 @@
|
|
|
1117
1012
|
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)",
|
|
1118
1013
|
"Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)"
|
|
1119
1014
|
]
|
|
1120
|
-
}
|
|
1121
|
-
,
|
|
1015
|
+
},
|
|
1122
1016
|
{
|
|
1123
1017
|
"pattern": "ec2linkfinder",
|
|
1124
1018
|
"addition_date": "2012/03/22",
|
|
1125
1019
|
"instances": [
|
|
1126
1020
|
"ec2linkfinder"
|
|
1127
1021
|
]
|
|
1128
|
-
}
|
|
1129
|
-
,
|
|
1022
|
+
},
|
|
1130
1023
|
{
|
|
1131
1024
|
"pattern": "gslfbot",
|
|
1132
1025
|
"addition_date": "2012/04/03",
|
|
1133
1026
|
"instances": []
|
|
1134
|
-
}
|
|
1135
|
-
,
|
|
1027
|
+
},
|
|
1136
1028
|
{
|
|
1137
1029
|
"pattern": "aiHitBot",
|
|
1138
1030
|
"addition_date": "2012/04/16",
|
|
1139
1031
|
"instances": [
|
|
1140
1032
|
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)"
|
|
1141
1033
|
]
|
|
1142
|
-
}
|
|
1143
|
-
,
|
|
1034
|
+
},
|
|
1144
1035
|
{
|
|
1145
1036
|
"pattern": "intelium_bot",
|
|
1146
1037
|
"addition_date": "2012/05/07",
|
|
1147
1038
|
"instances": []
|
|
1148
|
-
}
|
|
1149
|
-
,
|
|
1039
|
+
},
|
|
1150
1040
|
{
|
|
1151
1041
|
"pattern": "facebookexternalhit",
|
|
1152
1042
|
"addition_date": "2012/05/07",
|
|
@@ -1156,8 +1046,7 @@
|
|
|
1156
1046
|
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
|
|
1157
1047
|
],
|
|
1158
1048
|
"url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/"
|
|
1159
|
-
}
|
|
1160
|
-
,
|
|
1049
|
+
},
|
|
1161
1050
|
{
|
|
1162
1051
|
"pattern": "Yeti",
|
|
1163
1052
|
"addition_date": "2012/05/07",
|
|
@@ -1165,22 +1054,19 @@
|
|
|
1165
1054
|
"instances": [
|
|
1166
1055
|
"Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"
|
|
1167
1056
|
]
|
|
1168
|
-
}
|
|
1169
|
-
,
|
|
1057
|
+
},
|
|
1170
1058
|
{
|
|
1171
1059
|
"pattern": "RetrevoPageAnalyzer",
|
|
1172
1060
|
"addition_date": "2012/05/07",
|
|
1173
1061
|
"instances": [
|
|
1174
1062
|
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"
|
|
1175
1063
|
]
|
|
1176
|
-
}
|
|
1177
|
-
,
|
|
1064
|
+
},
|
|
1178
1065
|
{
|
|
1179
1066
|
"pattern": "lb-spider",
|
|
1180
1067
|
"addition_date": "2012/05/07",
|
|
1181
1068
|
"instances": []
|
|
1182
|
-
}
|
|
1183
|
-
,
|
|
1069
|
+
},
|
|
1184
1070
|
{
|
|
1185
1071
|
"pattern": "Sogou",
|
|
1186
1072
|
"addition_date": "2012/05/13",
|
|
@@ -1190,21 +1076,18 @@
|
|
|
1190
1076
|
"Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
|
|
1191
1077
|
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
|
|
1192
1078
|
]
|
|
1193
|
-
}
|
|
1194
|
-
,
|
|
1079
|
+
},
|
|
1195
1080
|
{
|
|
1196
1081
|
"pattern": "lssbot",
|
|
1197
1082
|
"addition_date": "2012/05/15",
|
|
1198
1083
|
"instances": []
|
|
1199
|
-
}
|
|
1200
|
-
,
|
|
1084
|
+
},
|
|
1201
1085
|
{
|
|
1202
1086
|
"pattern": "careerbot",
|
|
1203
1087
|
"addition_date": "2012/05/23",
|
|
1204
1088
|
"url": "http://www.career-x.de/bot.html",
|
|
1205
1089
|
"instances": []
|
|
1206
|
-
}
|
|
1207
|
-
,
|
|
1090
|
+
},
|
|
1208
1091
|
{
|
|
1209
1092
|
"pattern": "wotbox",
|
|
1210
1093
|
"addition_date": "2012/06/12",
|
|
@@ -1213,15 +1096,13 @@
|
|
|
1213
1096
|
"Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",
|
|
1214
1097
|
"Wotbox/2.01 (+http://www.wotbox.com/bot/)"
|
|
1215
1098
|
]
|
|
1216
|
-
}
|
|
1217
|
-
,
|
|
1099
|
+
},
|
|
1218
1100
|
{
|
|
1219
1101
|
"pattern": "wocbot",
|
|
1220
1102
|
"addition_date": "2012/07/25",
|
|
1221
1103
|
"url": "http://www.wocodi.com/crawler",
|
|
1222
1104
|
"instances": []
|
|
1223
|
-
}
|
|
1224
|
-
,
|
|
1105
|
+
},
|
|
1225
1106
|
{
|
|
1226
1107
|
"pattern": "ichiro",
|
|
1227
1108
|
"addition_date": "2012/08/28",
|
|
@@ -1243,8 +1124,7 @@
|
|
|
1243
1124
|
"ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",
|
|
1244
1125
|
"ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"
|
|
1245
1126
|
]
|
|
1246
|
-
}
|
|
1247
|
-
,
|
|
1127
|
+
},
|
|
1248
1128
|
{
|
|
1249
1129
|
"pattern": "DuckDuckBot",
|
|
1250
1130
|
"addition_date": "2012/09/19",
|
|
@@ -1255,14 +1135,12 @@
|
|
|
1255
1135
|
"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)",
|
|
1256
1136
|
"'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'"
|
|
1257
1137
|
]
|
|
1258
|
-
}
|
|
1259
|
-
,
|
|
1138
|
+
},
|
|
1260
1139
|
{
|
|
1261
1140
|
"pattern": "lssrocketcrawler",
|
|
1262
1141
|
"addition_date": "2012/09/24",
|
|
1263
1142
|
"instances": []
|
|
1264
|
-
}
|
|
1265
|
-
,
|
|
1143
|
+
},
|
|
1266
1144
|
{
|
|
1267
1145
|
"pattern": "drupact",
|
|
1268
1146
|
"addition_date": "2012/09/27",
|
|
@@ -1270,45 +1148,38 @@
|
|
|
1270
1148
|
"instances": [
|
|
1271
1149
|
"drupact/0.7; http://www.arocom.de/drupact"
|
|
1272
1150
|
]
|
|
1273
|
-
}
|
|
1274
|
-
,
|
|
1151
|
+
},
|
|
1275
1152
|
{
|
|
1276
1153
|
"pattern": "webcompanycrawler",
|
|
1277
1154
|
"addition_date": "2012/10/03",
|
|
1278
1155
|
"instances": []
|
|
1279
|
-
}
|
|
1280
|
-
,
|
|
1156
|
+
},
|
|
1281
1157
|
{
|
|
1282
1158
|
"pattern": "acoonbot",
|
|
1283
1159
|
"addition_date": "2012/10/07",
|
|
1284
1160
|
"url": "http://www.acoon.de/robot.asp",
|
|
1285
1161
|
"instances": []
|
|
1286
|
-
}
|
|
1287
|
-
,
|
|
1162
|
+
},
|
|
1288
1163
|
{
|
|
1289
1164
|
"pattern": "openindexspider",
|
|
1290
1165
|
"addition_date": "2012/10/26",
|
|
1291
1166
|
"url": "http://www.openindex.io/en/webmasters/spider.html",
|
|
1292
1167
|
"instances": []
|
|
1293
|
-
}
|
|
1294
|
-
,
|
|
1168
|
+
},
|
|
1295
1169
|
{
|
|
1296
1170
|
"pattern": "gnam gnam spider",
|
|
1297
1171
|
"addition_date": "2012/10/31",
|
|
1298
1172
|
"instances": []
|
|
1299
|
-
}
|
|
1300
|
-
,
|
|
1173
|
+
},
|
|
1301
1174
|
{
|
|
1302
1175
|
"pattern": "web-archive-net\\.com\\.bot",
|
|
1303
1176
|
"instances": []
|
|
1304
|
-
}
|
|
1305
|
-
,
|
|
1177
|
+
},
|
|
1306
1178
|
{
|
|
1307
1179
|
"pattern": "backlinkcrawler",
|
|
1308
1180
|
"addition_date": "2013/01/04",
|
|
1309
1181
|
"instances": []
|
|
1310
|
-
}
|
|
1311
|
-
,
|
|
1182
|
+
},
|
|
1312
1183
|
{
|
|
1313
1184
|
"pattern": "coccoc",
|
|
1314
1185
|
"addition_date": "2013/01/04",
|
|
@@ -1326,8 +1197,7 @@
|
|
|
1326
1197
|
"coccoc/1.0 (http://help.coccoc.com/)",
|
|
1327
1198
|
"coccoc/1.0 (http://help.coccoc.vn/)"
|
|
1328
1199
|
]
|
|
1329
|
-
}
|
|
1330
|
-
,
|
|
1200
|
+
},
|
|
1331
1201
|
{
|
|
1332
1202
|
"pattern": "integromedb",
|
|
1333
1203
|
"addition_date": "2013/01/10",
|
|
@@ -1335,20 +1205,17 @@
|
|
|
1335
1205
|
"instances": [
|
|
1336
1206
|
"www.integromedb.org/Crawler"
|
|
1337
1207
|
]
|
|
1338
|
-
}
|
|
1339
|
-
,
|
|
1208
|
+
},
|
|
1340
1209
|
{
|
|
1341
1210
|
"pattern": "content crawler spider",
|
|
1342
1211
|
"addition_date": "2013/01/11",
|
|
1343
1212
|
"instances": []
|
|
1344
|
-
}
|
|
1345
|
-
,
|
|
1213
|
+
},
|
|
1346
1214
|
{
|
|
1347
1215
|
"pattern": "toplistbot",
|
|
1348
1216
|
"addition_date": "2013/02/05",
|
|
1349
1217
|
"instances": []
|
|
1350
|
-
}
|
|
1351
|
-
,
|
|
1218
|
+
},
|
|
1352
1219
|
{
|
|
1353
1220
|
"pattern": "it2media-domain-crawler",
|
|
1354
1221
|
"addition_date": "2013/03/12",
|
|
@@ -1356,14 +1223,12 @@
|
|
|
1356
1223
|
"it2media-domain-crawler/1.0 on crawler-prod.it2media.de",
|
|
1357
1224
|
"it2media-domain-crawler/2.0"
|
|
1358
1225
|
]
|
|
1359
|
-
}
|
|
1360
|
-
,
|
|
1226
|
+
},
|
|
1361
1227
|
{
|
|
1362
1228
|
"pattern": "ip-web-crawler\\.com",
|
|
1363
1229
|
"addition_date": "2013/03/22",
|
|
1364
1230
|
"instances": []
|
|
1365
|
-
}
|
|
1366
|
-
,
|
|
1231
|
+
},
|
|
1367
1232
|
{
|
|
1368
1233
|
"pattern": "siteexplorer\\.info",
|
|
1369
1234
|
"addition_date": "2013/05/01",
|
|
@@ -1371,14 +1236,12 @@
|
|
|
1371
1236
|
"Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",
|
|
1372
1237
|
"Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"
|
|
1373
1238
|
]
|
|
1374
|
-
}
|
|
1375
|
-
,
|
|
1239
|
+
},
|
|
1376
1240
|
{
|
|
1377
1241
|
"pattern": "elisabot",
|
|
1378
1242
|
"addition_date": "2013/06/27",
|
|
1379
1243
|
"instances": []
|
|
1380
|
-
}
|
|
1381
|
-
,
|
|
1244
|
+
},
|
|
1382
1245
|
{
|
|
1383
1246
|
"pattern": "proximic",
|
|
1384
1247
|
"addition_date": "2013/09/12",
|
|
@@ -1387,8 +1250,7 @@
|
|
|
1387
1250
|
"Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",
|
|
1388
1251
|
"Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"
|
|
1389
1252
|
]
|
|
1390
|
-
}
|
|
1391
|
-
,
|
|
1253
|
+
},
|
|
1392
1254
|
{
|
|
1393
1255
|
"pattern": "changedetection",
|
|
1394
1256
|
"addition_date": "2013/09/13",
|
|
@@ -1396,14 +1258,12 @@
|
|
|
1396
1258
|
"instances": [
|
|
1397
1259
|
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )"
|
|
1398
1260
|
]
|
|
1399
|
-
}
|
|
1400
|
-
,
|
|
1261
|
+
},
|
|
1401
1262
|
{
|
|
1402
1263
|
"pattern": "arabot",
|
|
1403
1264
|
"addition_date": "2013/10/09",
|
|
1404
1265
|
"instances": []
|
|
1405
|
-
}
|
|
1406
|
-
,
|
|
1266
|
+
},
|
|
1407
1267
|
{
|
|
1408
1268
|
"pattern": "WeSEE:Search",
|
|
1409
1269
|
"addition_date": "2013/11/18",
|
|
@@ -1411,21 +1271,18 @@
|
|
|
1411
1271
|
"WeSEE:Search",
|
|
1412
1272
|
"WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"
|
|
1413
1273
|
]
|
|
1414
|
-
}
|
|
1415
|
-
,
|
|
1274
|
+
},
|
|
1416
1275
|
{
|
|
1417
1276
|
"pattern": "niki-bot",
|
|
1418
1277
|
"addition_date": "2014/01/01",
|
|
1419
1278
|
"instances": []
|
|
1420
|
-
}
|
|
1421
|
-
,
|
|
1279
|
+
},
|
|
1422
1280
|
{
|
|
1423
1281
|
"pattern": "CrystalSemanticsBot",
|
|
1424
1282
|
"addition_date": "2014/02/17",
|
|
1425
1283
|
"url": "http://www.crystalsemantics.com/user-agent/",
|
|
1426
1284
|
"instances": []
|
|
1427
|
-
}
|
|
1428
|
-
,
|
|
1285
|
+
},
|
|
1429
1286
|
{
|
|
1430
1287
|
"pattern": "rogerbot",
|
|
1431
1288
|
"addition_date": "2014/02/28",
|
|
@@ -1445,8 +1302,7 @@
|
|
|
1445
1302
|
"rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",
|
|
1446
1303
|
"rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"
|
|
1447
1304
|
]
|
|
1448
|
-
}
|
|
1449
|
-
,
|
|
1305
|
+
},
|
|
1450
1306
|
{
|
|
1451
1307
|
"pattern": "360Spider",
|
|
1452
1308
|
"addition_date": "2014/03/14",
|
|
@@ -1463,8 +1319,7 @@
|
|
|
1463
1319
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",
|
|
1464
1320
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider"
|
|
1465
1321
|
]
|
|
1466
|
-
}
|
|
1467
|
-
,
|
|
1322
|
+
},
|
|
1468
1323
|
{
|
|
1469
1324
|
"pattern": "psbot",
|
|
1470
1325
|
"addition_date": "2014/03/31",
|
|
@@ -1474,15 +1329,13 @@
|
|
|
1474
1329
|
"psbot-page (+http://www.picsearch.com/bot.html)",
|
|
1475
1330
|
"psbot/0.1 (+http://www.picsearch.com/bot.html)"
|
|
1476
1331
|
]
|
|
1477
|
-
}
|
|
1478
|
-
,
|
|
1332
|
+
},
|
|
1479
1333
|
{
|
|
1480
1334
|
"pattern": "InterfaxScanBot",
|
|
1481
1335
|
"addition_date": "2014/03/31",
|
|
1482
1336
|
"url": "http://scan-interfax.ru",
|
|
1483
1337
|
"instances": []
|
|
1484
|
-
}
|
|
1485
|
-
,
|
|
1338
|
+
},
|
|
1486
1339
|
{
|
|
1487
1340
|
"pattern": "CC Metadata Scaper",
|
|
1488
1341
|
"addition_date": "2014/04/01",
|
|
@@ -1490,15 +1343,13 @@
|
|
|
1490
1343
|
"instances": [
|
|
1491
1344
|
"CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"
|
|
1492
1345
|
]
|
|
1493
|
-
}
|
|
1494
|
-
,
|
|
1346
|
+
},
|
|
1495
1347
|
{
|
|
1496
1348
|
"pattern": "g00g1e\\.net",
|
|
1497
1349
|
"addition_date": "2014/04/01",
|
|
1498
1350
|
"url": "http://www.g00g1e.net/",
|
|
1499
1351
|
"instances": []
|
|
1500
|
-
}
|
|
1501
|
-
,
|
|
1352
|
+
},
|
|
1502
1353
|
{
|
|
1503
1354
|
"pattern": "GrapeshotCrawler",
|
|
1504
1355
|
"addition_date": "2014/04/01",
|
|
@@ -1506,8 +1357,7 @@
|
|
|
1506
1357
|
"instances": [
|
|
1507
1358
|
"Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
|
|
1508
1359
|
]
|
|
1509
|
-
}
|
|
1510
|
-
,
|
|
1360
|
+
},
|
|
1511
1361
|
{
|
|
1512
1362
|
"pattern": "urlappendbot",
|
|
1513
1363
|
"addition_date": "2014/05/10",
|
|
@@ -1515,22 +1365,19 @@
|
|
|
1515
1365
|
"instances": [
|
|
1516
1366
|
"Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"
|
|
1517
1367
|
]
|
|
1518
|
-
}
|
|
1519
|
-
,
|
|
1368
|
+
},
|
|
1520
1369
|
{
|
|
1521
1370
|
"pattern": "brainobot",
|
|
1522
1371
|
"addition_date": "2014/06/24",
|
|
1523
1372
|
"instances": []
|
|
1524
|
-
}
|
|
1525
|
-
,
|
|
1373
|
+
},
|
|
1526
1374
|
{
|
|
1527
1375
|
"pattern": "fr-crawler",
|
|
1528
1376
|
"addition_date": "2014/07/31",
|
|
1529
1377
|
"instances": [
|
|
1530
1378
|
"Mozilla/5.0 (compatible; fr-crawler/1.1)"
|
|
1531
1379
|
]
|
|
1532
|
-
}
|
|
1533
|
-
,
|
|
1380
|
+
},
|
|
1534
1381
|
{
|
|
1535
1382
|
"pattern": "binlar",
|
|
1536
1383
|
"addition_date": "2014/09/12",
|
|
@@ -1541,16 +1388,14 @@
|
|
|
1541
1388
|
"binlar_2.6.3 phanendra_kalapala@McAfee.com",
|
|
1542
1389
|
"binlar_2.6.3 test@mgmt.mic"
|
|
1543
1390
|
]
|
|
1544
|
-
}
|
|
1545
|
-
,
|
|
1391
|
+
},
|
|
1546
1392
|
{
|
|
1547
1393
|
"pattern": "SimpleCrawler",
|
|
1548
1394
|
"addition_date": "2014/09/12",
|
|
1549
1395
|
"instances": [
|
|
1550
1396
|
"SimpleCrawler/0.1"
|
|
1551
1397
|
]
|
|
1552
|
-
}
|
|
1553
|
-
,
|
|
1398
|
+
},
|
|
1554
1399
|
{
|
|
1555
1400
|
"pattern": "Twitterbot",
|
|
1556
1401
|
"addition_date": "2014/09/12",
|
|
@@ -1559,8 +1404,7 @@
|
|
|
1559
1404
|
"Twitterbot/0.1",
|
|
1560
1405
|
"Twitterbot/1.0"
|
|
1561
1406
|
]
|
|
1562
|
-
}
|
|
1563
|
-
,
|
|
1407
|
+
},
|
|
1564
1408
|
{
|
|
1565
1409
|
"pattern": "cXensebot",
|
|
1566
1410
|
"addition_date": "2014/10/05",
|
|
@@ -1568,8 +1412,7 @@
|
|
|
1568
1412
|
"cXensebot/1.1a"
|
|
1569
1413
|
],
|
|
1570
1414
|
"url": "http://www.cxense.com/bot.html"
|
|
1571
|
-
}
|
|
1572
|
-
,
|
|
1415
|
+
},
|
|
1573
1416
|
{
|
|
1574
1417
|
"pattern": "smtbot",
|
|
1575
1418
|
"addition_date": "2014/10/04",
|
|
@@ -1581,8 +1424,7 @@
|
|
|
1581
1424
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)"
|
|
1582
1425
|
],
|
|
1583
1426
|
"url": "http://www.similartech.com/smtbot"
|
|
1584
|
-
}
|
|
1585
|
-
,
|
|
1427
|
+
},
|
|
1586
1428
|
{
|
|
1587
1429
|
"pattern": "bnf\\.fr_bot",
|
|
1588
1430
|
"addition_date": "2014/11/18",
|
|
@@ -1591,8 +1433,7 @@
|
|
|
1591
1433
|
"Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)",
|
|
1592
1434
|
"Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"
|
|
1593
1435
|
]
|
|
1594
|
-
}
|
|
1595
|
-
,
|
|
1436
|
+
},
|
|
1596
1437
|
{
|
|
1597
1438
|
"pattern": "A6-Indexer",
|
|
1598
1439
|
"addition_date": "2014/12/05",
|
|
@@ -1600,8 +1441,7 @@
|
|
|
1600
1441
|
"instances": [
|
|
1601
1442
|
"A6-Indexer"
|
|
1602
1443
|
]
|
|
1603
|
-
}
|
|
1604
|
-
,
|
|
1444
|
+
},
|
|
1605
1445
|
{
|
|
1606
1446
|
"pattern": "ADmantX",
|
|
1607
1447
|
"addition_date": "2014/12/05",
|
|
@@ -1609,8 +1449,7 @@
|
|
|
1609
1449
|
"instances": [
|
|
1610
1450
|
"ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"
|
|
1611
1451
|
]
|
|
1612
|
-
}
|
|
1613
|
-
,
|
|
1452
|
+
},
|
|
1614
1453
|
{
|
|
1615
1454
|
"pattern": "Facebot",
|
|
1616
1455
|
"url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",
|
|
@@ -1618,16 +1457,14 @@
|
|
|
1618
1457
|
"instances": [
|
|
1619
1458
|
"Facebot/1.0"
|
|
1620
1459
|
]
|
|
1621
|
-
}
|
|
1622
|
-
,
|
|
1460
|
+
},
|
|
1623
1461
|
{
|
|
1624
1462
|
"pattern": "OrangeBot\\/",
|
|
1625
1463
|
"instances": [
|
|
1626
1464
|
"Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"
|
|
1627
1465
|
],
|
|
1628
1466
|
"addition_date": "2015/01/12"
|
|
1629
|
-
}
|
|
1630
|
-
,
|
|
1467
|
+
},
|
|
1631
1468
|
{
|
|
1632
1469
|
"pattern": "memorybot",
|
|
1633
1470
|
"url": "http://mignify.com/bot.htm",
|
|
@@ -1635,8 +1472,7 @@
|
|
|
1635
1472
|
"Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"
|
|
1636
1473
|
],
|
|
1637
1474
|
"addition_date": "2015/02/01"
|
|
1638
|
-
}
|
|
1639
|
-
,
|
|
1475
|
+
},
|
|
1640
1476
|
{
|
|
1641
1477
|
"pattern": "AdvBot",
|
|
1642
1478
|
"url": "http://advbot.net/bot.html",
|
|
@@ -1644,8 +1480,7 @@
|
|
|
1644
1480
|
"Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"
|
|
1645
1481
|
],
|
|
1646
1482
|
"addition_date": "2015/02/01"
|
|
1647
|
-
}
|
|
1648
|
-
,
|
|
1483
|
+
},
|
|
1649
1484
|
{
|
|
1650
1485
|
"pattern": "MegaIndex",
|
|
1651
1486
|
"url": "https://www.megaindex.ru/?tab=linkAnalyze",
|
|
@@ -1654,8 +1489,7 @@
|
|
|
1654
1489
|
"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)"
|
|
1655
1490
|
],
|
|
1656
1491
|
"addition_date": "2015/03/28"
|
|
1657
|
-
}
|
|
1658
|
-
,
|
|
1492
|
+
},
|
|
1659
1493
|
{
|
|
1660
1494
|
"pattern": "SemanticScholarBot",
|
|
1661
1495
|
"url": "https://www.semanticscholar.org/crawler",
|
|
@@ -1664,8 +1498,7 @@
|
|
|
1664
1498
|
"Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)"
|
|
1665
1499
|
],
|
|
1666
1500
|
"addition_date": "2015/03/28"
|
|
1667
|
-
}
|
|
1668
|
-
,
|
|
1501
|
+
},
|
|
1669
1502
|
{
|
|
1670
1503
|
"pattern": "ltx71",
|
|
1671
1504
|
"url": "http://ltx71.com/",
|
|
@@ -1673,8 +1506,7 @@
|
|
|
1673
1506
|
"ltx71 - (http://ltx71.com/)"
|
|
1674
1507
|
],
|
|
1675
1508
|
"addition_date": "2015/04/04"
|
|
1676
|
-
}
|
|
1677
|
-
,
|
|
1509
|
+
},
|
|
1678
1510
|
{
|
|
1679
1511
|
"pattern": "nerdybot",
|
|
1680
1512
|
"url": "http://nerdybot.com/",
|
|
@@ -1682,8 +1514,7 @@
|
|
|
1682
1514
|
"nerdybot"
|
|
1683
1515
|
],
|
|
1684
1516
|
"addition_date": "2015/04/05"
|
|
1685
|
-
}
|
|
1686
|
-
,
|
|
1517
|
+
},
|
|
1687
1518
|
{
|
|
1688
1519
|
"pattern": "xovibot",
|
|
1689
1520
|
"url": "http://www.xovibot.net/",
|
|
@@ -1691,8 +1522,7 @@
|
|
|
1691
1522
|
"Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"
|
|
1692
1523
|
],
|
|
1693
1524
|
"addition_date": "2015/04/05"
|
|
1694
|
-
}
|
|
1695
|
-
,
|
|
1525
|
+
},
|
|
1696
1526
|
{
|
|
1697
1527
|
"pattern": "BUbiNG",
|
|
1698
1528
|
"url": "http://law.di.unimi.it/BUbiNG.html",
|
|
@@ -1700,8 +1530,7 @@
|
|
|
1700
1530
|
"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"
|
|
1701
1531
|
],
|
|
1702
1532
|
"addition_date": "2015/04/06"
|
|
1703
|
-
}
|
|
1704
|
-
,
|
|
1533
|
+
},
|
|
1705
1534
|
{
|
|
1706
1535
|
"pattern": "Qwantify",
|
|
1707
1536
|
"url": "https://www.qwant.com/",
|
|
@@ -1712,12 +1541,13 @@
|
|
|
1712
1541
|
"Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)"
|
|
1713
1542
|
],
|
|
1714
1543
|
"addition_date": "2015/04/06"
|
|
1715
|
-
}
|
|
1716
|
-
,
|
|
1544
|
+
},
|
|
1717
1545
|
{
|
|
1718
1546
|
"pattern": "archive\\.org_bot",
|
|
1719
1547
|
"url": "http://www.archive.org/details/archive.org_bot",
|
|
1720
|
-
"depends_on": [
|
|
1548
|
+
"depends_on": [
|
|
1549
|
+
"heritrix"
|
|
1550
|
+
],
|
|
1721
1551
|
"instances": [
|
|
1722
1552
|
"Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)",
|
|
1723
1553
|
"Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)",
|
|
@@ -1727,8 +1557,7 @@
|
|
|
1727
1557
|
"Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)"
|
|
1728
1558
|
],
|
|
1729
1559
|
"addition_date": "2015/04/14"
|
|
1730
|
-
}
|
|
1731
|
-
,
|
|
1560
|
+
},
|
|
1732
1561
|
{
|
|
1733
1562
|
"pattern": "Applebot",
|
|
1734
1563
|
"url": "http://www.apple.com/go/applebot",
|
|
@@ -1740,8 +1569,7 @@
|
|
|
1740
1569
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
|
|
1741
1570
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"
|
|
1742
1571
|
]
|
|
1743
|
-
}
|
|
1744
|
-
,
|
|
1572
|
+
},
|
|
1745
1573
|
{
|
|
1746
1574
|
"pattern": "TweetmemeBot",
|
|
1747
1575
|
"url": "http://datasift.com/bot.html",
|
|
@@ -1749,8 +1577,7 @@
|
|
|
1749
1577
|
"Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"
|
|
1750
1578
|
],
|
|
1751
1579
|
"addition_date": "2015/04/15"
|
|
1752
|
-
}
|
|
1753
|
-
,
|
|
1580
|
+
},
|
|
1754
1581
|
{
|
|
1755
1582
|
"pattern": "crawler4j",
|
|
1756
1583
|
"url": "https://github.com/yasserg/crawler4j",
|
|
@@ -1759,8 +1586,7 @@
|
|
|
1759
1586
|
"crawler4j (https://github.com/yasserg/crawler4j/)"
|
|
1760
1587
|
],
|
|
1761
1588
|
"addition_date": "2015/05/07"
|
|
1762
|
-
}
|
|
1763
|
-
,
|
|
1589
|
+
},
|
|
1764
1590
|
{
|
|
1765
1591
|
"pattern": "findxbot",
|
|
1766
1592
|
"url": "http://www.findxbot.com",
|
|
@@ -1768,8 +1594,7 @@
|
|
|
1768
1594
|
"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"
|
|
1769
1595
|
],
|
|
1770
1596
|
"addition_date": "2015/05/07"
|
|
1771
|
-
}
|
|
1772
|
-
,
|
|
1597
|
+
},
|
|
1773
1598
|
{
|
|
1774
1599
|
"pattern": "S[eE][mM]rushBot",
|
|
1775
1600
|
"url": "http://www.semrush.com/bot.html",
|
|
@@ -1784,8 +1609,7 @@
|
|
|
1784
1609
|
"SEMrushBot"
|
|
1785
1610
|
],
|
|
1786
1611
|
"addition_date": "2015/05/26"
|
|
1787
|
-
}
|
|
1788
|
-
,
|
|
1612
|
+
},
|
|
1789
1613
|
{
|
|
1790
1614
|
"pattern": "yoozBot",
|
|
1791
1615
|
"url": "http://yooz.ir",
|
|
@@ -1793,8 +1617,7 @@
|
|
|
1793
1617
|
"Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"
|
|
1794
1618
|
],
|
|
1795
1619
|
"addition_date": "2015/05/26"
|
|
1796
|
-
}
|
|
1797
|
-
,
|
|
1620
|
+
},
|
|
1798
1621
|
{
|
|
1799
1622
|
"pattern": "lipperhey",
|
|
1800
1623
|
"url": "http://www.lipperhey.com/",
|
|
@@ -1805,8 +1628,7 @@
|
|
|
1805
1628
|
"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"
|
|
1806
1629
|
],
|
|
1807
1630
|
"addition_date": "2015/08/26"
|
|
1808
|
-
}
|
|
1809
|
-
,
|
|
1631
|
+
},
|
|
1810
1632
|
{
|
|
1811
1633
|
"pattern": "Y!J",
|
|
1812
1634
|
"url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6",
|
|
@@ -1819,8 +1641,7 @@
|
|
|
1819
1641
|
"Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))"
|
|
1820
1642
|
],
|
|
1821
1643
|
"addition_date": "2015/05/26"
|
|
1822
|
-
}
|
|
1823
|
-
,
|
|
1644
|
+
},
|
|
1824
1645
|
{
|
|
1825
1646
|
"pattern": "Domain Re-Animator Bot",
|
|
1826
1647
|
"url": "http://domainreanimator.com",
|
|
@@ -1828,8 +1649,7 @@
|
|
|
1828
1649
|
"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
|
|
1829
1650
|
],
|
|
1830
1651
|
"addition_date": "2015/04/14"
|
|
1831
|
-
}
|
|
1832
|
-
,
|
|
1652
|
+
},
|
|
1833
1653
|
{
|
|
1834
1654
|
"pattern": "AddThis",
|
|
1835
1655
|
"url": "https://www.addthis.com",
|
|
@@ -1837,8 +1657,7 @@
|
|
|
1837
1657
|
"AddThis.com robot tech.support@clearspring.com"
|
|
1838
1658
|
],
|
|
1839
1659
|
"addition_date": "2015/06/02"
|
|
1840
|
-
}
|
|
1841
|
-
,
|
|
1660
|
+
},
|
|
1842
1661
|
{
|
|
1843
1662
|
"pattern": "Screaming Frog SEO Spider",
|
|
1844
1663
|
"url": "http://www.screamingfrog.co.uk/seo-spider",
|
|
@@ -1846,8 +1665,7 @@
|
|
|
1846
1665
|
"Screaming Frog SEO Spider/5.1"
|
|
1847
1666
|
],
|
|
1848
1667
|
"addition_date": "2016/01/08"
|
|
1849
|
-
}
|
|
1850
|
-
,
|
|
1668
|
+
},
|
|
1851
1669
|
{
|
|
1852
1670
|
"pattern": "MetaURI",
|
|
1853
1671
|
"url": "http://www.useragentstring.com/MetaURI_id_17683.php",
|
|
@@ -1855,8 +1673,7 @@
|
|
|
1855
1673
|
"MetaURI API/2.0 +metauri.com"
|
|
1856
1674
|
],
|
|
1857
1675
|
"addition_date": "2016/01/02"
|
|
1858
|
-
}
|
|
1859
|
-
,
|
|
1676
|
+
},
|
|
1860
1677
|
{
|
|
1861
1678
|
"pattern": "Scrapy",
|
|
1862
1679
|
"url": "http://scrapy.org/",
|
|
@@ -1864,8 +1681,7 @@
|
|
|
1864
1681
|
"Scrapy/1.0.3 (+http://scrapy.org)"
|
|
1865
1682
|
],
|
|
1866
1683
|
"addition_date": "2016/01/02"
|
|
1867
|
-
}
|
|
1868
|
-
,
|
|
1684
|
+
},
|
|
1869
1685
|
{
|
|
1870
1686
|
"pattern": "Livelap[bB]ot",
|
|
1871
1687
|
"url": "http://site.livelap.com/crawler",
|
|
@@ -1874,8 +1690,7 @@
|
|
|
1874
1690
|
"Livelapbot/0.1"
|
|
1875
1691
|
],
|
|
1876
1692
|
"addition_date": "2016/01/02"
|
|
1877
|
-
}
|
|
1878
|
-
,
|
|
1693
|
+
},
|
|
1879
1694
|
{
|
|
1880
1695
|
"pattern": "OpenHoseBot",
|
|
1881
1696
|
"url": "http://www.openhose.org/bot.html",
|
|
@@ -1883,8 +1698,7 @@
|
|
|
1883
1698
|
"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"
|
|
1884
1699
|
],
|
|
1885
1700
|
"addition_date": "2016/01/02"
|
|
1886
|
-
}
|
|
1887
|
-
,
|
|
1701
|
+
},
|
|
1888
1702
|
{
|
|
1889
1703
|
"pattern": "CapsuleChecker",
|
|
1890
1704
|
"url": "http://www.capsulink.com/about",
|
|
@@ -1892,8 +1706,7 @@
|
|
|
1892
1706
|
"CapsuleChecker (http://www.capsulink.com/)"
|
|
1893
1707
|
],
|
|
1894
1708
|
"addition_date": "2016/01/02"
|
|
1895
|
-
}
|
|
1896
|
-
,
|
|
1709
|
+
},
|
|
1897
1710
|
{
|
|
1898
1711
|
"pattern": "collection@infegy\\.com",
|
|
1899
1712
|
"url": "http://infegy.com/",
|
|
@@ -1901,8 +1714,7 @@
|
|
|
1901
1714
|
"Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"
|
|
1902
1715
|
],
|
|
1903
1716
|
"addition_date": "2016/01/03"
|
|
1904
|
-
}
|
|
1905
|
-
,
|
|
1717
|
+
},
|
|
1906
1718
|
{
|
|
1907
1719
|
"pattern": "IstellaBot",
|
|
1908
1720
|
"url": "http://www.tiscali.it/",
|
|
@@ -1910,8 +1722,7 @@
|
|
|
1910
1722
|
"Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"
|
|
1911
1723
|
],
|
|
1912
1724
|
"addition_date": "2016/01/09"
|
|
1913
|
-
}
|
|
1914
|
-
,
|
|
1725
|
+
},
|
|
1915
1726
|
{
|
|
1916
1727
|
"pattern": "DeuSu\\/",
|
|
1917
1728
|
"addition_date": "2016/01/23",
|
|
@@ -1920,14 +1731,12 @@
|
|
|
1920
1731
|
"Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",
|
|
1921
1732
|
"Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"
|
|
1922
1733
|
]
|
|
1923
|
-
}
|
|
1924
|
-
,
|
|
1734
|
+
},
|
|
1925
1735
|
{
|
|
1926
1736
|
"pattern": "betaBot",
|
|
1927
1737
|
"addition_date": "2016/01/23",
|
|
1928
1738
|
"instances": []
|
|
1929
|
-
}
|
|
1930
|
-
,
|
|
1739
|
+
},
|
|
1931
1740
|
{
|
|
1932
1741
|
"pattern": "Cliqzbot\\/",
|
|
1933
1742
|
"addition_date": "2016/01/23",
|
|
@@ -1939,8 +1748,7 @@
|
|
|
1939
1748
|
"Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",
|
|
1940
1749
|
"Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"
|
|
1941
1750
|
]
|
|
1942
|
-
}
|
|
1943
|
-
,
|
|
1751
|
+
},
|
|
1944
1752
|
{
|
|
1945
1753
|
"pattern": "MojeekBot\\/",
|
|
1946
1754
|
"addition_date": "2016/01/23",
|
|
@@ -1953,8 +1761,7 @@
|
|
|
1953
1761
|
"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",
|
|
1954
1762
|
"Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"
|
|
1955
1763
|
]
|
|
1956
|
-
}
|
|
1957
|
-
,
|
|
1764
|
+
},
|
|
1958
1765
|
{
|
|
1959
1766
|
"pattern": "netEstate NE Crawler",
|
|
1960
1767
|
"addition_date": "2016/01/23",
|
|
@@ -1963,8 +1770,7 @@
|
|
|
1963
1770
|
"netEstate NE Crawler (+http://www.sengine.info/)",
|
|
1964
1771
|
"netEstate NE Crawler (+http://www.website-datenbank.de/)"
|
|
1965
1772
|
]
|
|
1966
|
-
}
|
|
1967
|
-
,
|
|
1773
|
+
},
|
|
1968
1774
|
{
|
|
1969
1775
|
"pattern": "SafeSearch microdata crawler",
|
|
1970
1776
|
"addition_date": "2016/01/23",
|
|
@@ -1972,8 +1778,7 @@
|
|
|
1972
1778
|
"instances": [
|
|
1973
1779
|
"SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"
|
|
1974
1780
|
]
|
|
1975
|
-
}
|
|
1976
|
-
,
|
|
1781
|
+
},
|
|
1977
1782
|
{
|
|
1978
1783
|
"pattern": "Gluten Free Crawler\\/",
|
|
1979
1784
|
"addition_date": "2016/01/23",
|
|
@@ -1981,8 +1786,7 @@
|
|
|
1981
1786
|
"instances": [
|
|
1982
1787
|
"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"
|
|
1983
1788
|
]
|
|
1984
|
-
}
|
|
1985
|
-
,
|
|
1789
|
+
},
|
|
1986
1790
|
{
|
|
1987
1791
|
"pattern": "Sonic",
|
|
1988
1792
|
"addition_date": "2016/02/08",
|
|
@@ -1992,8 +1796,7 @@
|
|
|
1992
1796
|
"Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",
|
|
1993
1797
|
"Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"
|
|
1994
1798
|
]
|
|
1995
|
-
}
|
|
1996
|
-
,
|
|
1799
|
+
},
|
|
1997
1800
|
{
|
|
1998
1801
|
"pattern": "Sysomos",
|
|
1999
1802
|
"addition_date": "2016/02/08",
|
|
@@ -2001,15 +1804,13 @@
|
|
|
2001
1804
|
"instances": [
|
|
2002
1805
|
"Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"
|
|
2003
1806
|
]
|
|
2004
|
-
}
|
|
2005
|
-
,
|
|
1807
|
+
},
|
|
2006
1808
|
{
|
|
2007
1809
|
"pattern": "Trove",
|
|
2008
1810
|
"addition_date": "2016/02/08",
|
|
2009
1811
|
"url": "http://www.trove.com",
|
|
2010
1812
|
"instances": []
|
|
2011
|
-
}
|
|
2012
|
-
,
|
|
1813
|
+
},
|
|
2013
1814
|
{
|
|
2014
1815
|
"pattern": "deadlinkchecker",
|
|
2015
1816
|
"addition_date": "2016/02/08",
|
|
@@ -2019,8 +1820,7 @@
|
|
|
2019
1820
|
"www.deadlinkchecker.com XMLHTTP/1.0",
|
|
2020
1821
|
"www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"
|
|
2021
1822
|
]
|
|
2022
|
-
}
|
|
2023
|
-
,
|
|
1823
|
+
},
|
|
2024
1824
|
{
|
|
2025
1825
|
"pattern": "Slack-ImgProxy",
|
|
2026
1826
|
"addition_date": "2016/04/25",
|
|
@@ -2033,8 +1833,7 @@
|
|
|
2033
1833
|
"Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",
|
|
2034
1834
|
"Slack-ImgProxy 149 (+https://api.slack.com/robots)"
|
|
2035
1835
|
]
|
|
2036
|
-
}
|
|
2037
|
-
,
|
|
1836
|
+
},
|
|
2038
1837
|
{
|
|
2039
1838
|
"pattern": "Embedly",
|
|
2040
1839
|
"addition_date": "2016/04/25",
|
|
@@ -2044,8 +1843,7 @@
|
|
|
2044
1843
|
"Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",
|
|
2045
1844
|
"Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"
|
|
2046
1845
|
]
|
|
2047
|
-
}
|
|
2048
|
-
,
|
|
1846
|
+
},
|
|
2049
1847
|
{
|
|
2050
1848
|
"pattern": "RankActiveLinkBot",
|
|
2051
1849
|
"addition_date": "2016/06/20",
|
|
@@ -2053,8 +1851,7 @@
|
|
|
2053
1851
|
"instances": [
|
|
2054
1852
|
"Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"
|
|
2055
1853
|
]
|
|
2056
|
-
}
|
|
2057
|
-
,
|
|
1854
|
+
},
|
|
2058
1855
|
{
|
|
2059
1856
|
"pattern": "iskanie",
|
|
2060
1857
|
"addition_date": "2016/09/02",
|
|
@@ -2062,8 +1859,7 @@
|
|
|
2062
1859
|
"instances": [
|
|
2063
1860
|
"iskanie (+http://www.iskanie.com)"
|
|
2064
1861
|
]
|
|
2065
|
-
}
|
|
2066
|
-
,
|
|
1862
|
+
},
|
|
2067
1863
|
{
|
|
2068
1864
|
"pattern": "SafeDNSBot",
|
|
2069
1865
|
"addition_date": "2016/09/10",
|
|
@@ -2071,16 +1867,14 @@
|
|
|
2071
1867
|
"instances": [
|
|
2072
1868
|
"SafeDNSBot (https://www.safedns.com/searchbot)"
|
|
2073
1869
|
]
|
|
2074
|
-
}
|
|
2075
|
-
,
|
|
1870
|
+
},
|
|
2076
1871
|
{
|
|
2077
1872
|
"pattern": "SkypeUriPreview",
|
|
2078
1873
|
"addition_date": "2016/10/10",
|
|
2079
1874
|
"instances": [
|
|
2080
1875
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"
|
|
2081
1876
|
]
|
|
2082
|
-
}
|
|
2083
|
-
,
|
|
1877
|
+
},
|
|
2084
1878
|
{
|
|
2085
1879
|
"pattern": "Veoozbot",
|
|
2086
1880
|
"addition_date": "2016/11/03",
|
|
@@ -2088,8 +1882,7 @@
|
|
|
2088
1882
|
"instances": [
|
|
2089
1883
|
"Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"
|
|
2090
1884
|
]
|
|
2091
|
-
}
|
|
2092
|
-
,
|
|
1885
|
+
},
|
|
2093
1886
|
{
|
|
2094
1887
|
"pattern": "Slackbot",
|
|
2095
1888
|
"addition_date": "2016/11/03",
|
|
@@ -2099,8 +1892,7 @@
|
|
|
2099
1892
|
"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
|
|
2100
1893
|
"Slackbot 1.0 (+https://api.slack.com/robots)"
|
|
2101
1894
|
]
|
|
2102
|
-
}
|
|
2103
|
-
,
|
|
1895
|
+
},
|
|
2104
1896
|
{
|
|
2105
1897
|
"pattern": "redditbot",
|
|
2106
1898
|
"addition_date": "2016/11/03",
|
|
@@ -2108,8 +1900,7 @@
|
|
|
2108
1900
|
"instances": [
|
|
2109
1901
|
"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"
|
|
2110
1902
|
]
|
|
2111
|
-
}
|
|
2112
|
-
,
|
|
1903
|
+
},
|
|
2113
1904
|
{
|
|
2114
1905
|
"pattern": "datagnionbot",
|
|
2115
1906
|
"addition_date": "2016/11/03",
|
|
@@ -2117,8 +1908,7 @@
|
|
|
2117
1908
|
"instances": [
|
|
2118
1909
|
"datagnionbot (+http://www.datagnion.com/bot.html)"
|
|
2119
1910
|
]
|
|
2120
|
-
}
|
|
2121
|
-
,
|
|
1911
|
+
},
|
|
2122
1912
|
{
|
|
2123
1913
|
"pattern": "Google-Adwords-Instant",
|
|
2124
1914
|
"addition_date": "2016/11/03",
|
|
@@ -2126,8 +1916,7 @@
|
|
|
2126
1916
|
"instances": [
|
|
2127
1917
|
"Google-Adwords-Instant (+http://www.google.com/adsbot.html)"
|
|
2128
1918
|
]
|
|
2129
|
-
}
|
|
2130
|
-
,
|
|
1919
|
+
},
|
|
2131
1920
|
{
|
|
2132
1921
|
"pattern": "adbeat_bot",
|
|
2133
1922
|
"addition_date": "2016/11/04",
|
|
@@ -2135,8 +1924,7 @@
|
|
|
2135
1924
|
"Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",
|
|
2136
1925
|
"adbeat_bot"
|
|
2137
1926
|
]
|
|
2138
|
-
}
|
|
2139
|
-
,
|
|
1927
|
+
},
|
|
2140
1928
|
{
|
|
2141
1929
|
"pattern": "WhatsApp",
|
|
2142
1930
|
"addition_date": "2016/11/15",
|
|
@@ -2168,16 +1956,14 @@
|
|
|
2168
1956
|
"WhatsApp/2.19.308 A",
|
|
2169
1957
|
"WhatsApp/2.19.330 A"
|
|
2170
1958
|
]
|
|
2171
|
-
}
|
|
2172
|
-
,
|
|
1959
|
+
},
|
|
2173
1960
|
{
|
|
2174
1961
|
"pattern": "contxbot",
|
|
2175
1962
|
"addition_date": "2017/02/25",
|
|
2176
1963
|
"instances": [
|
|
2177
1964
|
"Mozilla/5.0 (compatible;contxbot/1.0)"
|
|
2178
1965
|
]
|
|
2179
|
-
}
|
|
2180
|
-
,
|
|
1966
|
+
},
|
|
2181
1967
|
{
|
|
2182
1968
|
"pattern": "pinterest\\.com\\/bot",
|
|
2183
1969
|
"addition_date": "2017/03/03",
|
|
@@ -2186,8 +1972,7 @@
|
|
|
2186
1972
|
"Pinterest/0.2 (+http://www.pinterest.com/bot.html)"
|
|
2187
1973
|
],
|
|
2188
1974
|
"url": "http://www.pinterest.com/bot.html"
|
|
2189
|
-
}
|
|
2190
|
-
,
|
|
1975
|
+
},
|
|
2191
1976
|
{
|
|
2192
1977
|
"pattern": "electricmonk",
|
|
2193
1978
|
"addition_date": "2017/03/04",
|
|
@@ -2195,8 +1980,7 @@
|
|
|
2195
1980
|
"Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"
|
|
2196
1981
|
],
|
|
2197
1982
|
"url": "https://www.duedil.com/our-crawler/"
|
|
2198
|
-
}
|
|
2199
|
-
,
|
|
1983
|
+
},
|
|
2200
1984
|
{
|
|
2201
1985
|
"pattern": "GarlikCrawler",
|
|
2202
1986
|
"addition_date": "2017/03/18",
|
|
@@ -2204,8 +1988,7 @@
|
|
|
2204
1988
|
"GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"
|
|
2205
1989
|
],
|
|
2206
1990
|
"url": "http://garlik.com/"
|
|
2207
|
-
}
|
|
2208
|
-
,
|
|
1991
|
+
},
|
|
2209
1992
|
{
|
|
2210
1993
|
"pattern": "BingPreview\\/",
|
|
2211
1994
|
"addition_date": "2017/04/23",
|
|
@@ -2217,8 +2000,7 @@
|
|
|
2217
2000
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)",
|
|
2218
2001
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"
|
|
2219
2002
|
]
|
|
2220
|
-
}
|
|
2221
|
-
,
|
|
2003
|
+
},
|
|
2222
2004
|
{
|
|
2223
2005
|
"pattern": "vebidoobot",
|
|
2224
2006
|
"addition_date": "2017/05/08",
|
|
@@ -2226,8 +2008,7 @@
|
|
|
2226
2008
|
"Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"
|
|
2227
2009
|
],
|
|
2228
2010
|
"url": "https://blog.vebidoo.de/vebidoobot/"
|
|
2229
|
-
}
|
|
2230
|
-
,
|
|
2011
|
+
},
|
|
2231
2012
|
{
|
|
2232
2013
|
"pattern": "FemtosearchBot",
|
|
2233
2014
|
"addition_date": "2017/05/16",
|
|
@@ -2235,8 +2016,7 @@
|
|
|
2235
2016
|
"Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"
|
|
2236
2017
|
],
|
|
2237
2018
|
"url": "http://femtosearch.com"
|
|
2238
|
-
}
|
|
2239
|
-
,
|
|
2019
|
+
},
|
|
2240
2020
|
{
|
|
2241
2021
|
"pattern": "Yahoo Link Preview",
|
|
2242
2022
|
"addition_date": "2017/06/28",
|
|
@@ -2244,8 +2024,7 @@
|
|
|
2244
2024
|
"Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"
|
|
2245
2025
|
],
|
|
2246
2026
|
"url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"
|
|
2247
|
-
}
|
|
2248
|
-
,
|
|
2027
|
+
},
|
|
2249
2028
|
{
|
|
2250
2029
|
"pattern": "MetaJobBot",
|
|
2251
2030
|
"addition_date": "2017/08/16",
|
|
@@ -2253,8 +2032,7 @@
|
|
|
2253
2032
|
"Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"
|
|
2254
2033
|
],
|
|
2255
2034
|
"url": "http://www.metajob.de/the/crawler"
|
|
2256
|
-
}
|
|
2257
|
-
,
|
|
2035
|
+
},
|
|
2258
2036
|
{
|
|
2259
2037
|
"pattern": "DomainStatsBot",
|
|
2260
2038
|
"addition_date": "2017/08/16",
|
|
@@ -2262,8 +2040,7 @@
|
|
|
2262
2040
|
"DomainStatsBot/1.0 (http://domainstats.io/our-bot)"
|
|
2263
2041
|
],
|
|
2264
2042
|
"url": "http://domainstats.io/our-bot"
|
|
2265
|
-
}
|
|
2266
|
-
,
|
|
2043
|
+
},
|
|
2267
2044
|
{
|
|
2268
2045
|
"pattern": "mindUpBot",
|
|
2269
2046
|
"addition_date": "2017/08/16",
|
|
@@ -2271,8 +2048,7 @@
|
|
|
2271
2048
|
"mindUpBot (datenbutler.de)"
|
|
2272
2049
|
],
|
|
2273
2050
|
"url": "http://www.datenbutler.de/"
|
|
2274
|
-
}
|
|
2275
|
-
,
|
|
2051
|
+
},
|
|
2276
2052
|
{
|
|
2277
2053
|
"pattern": "Daum\\/",
|
|
2278
2054
|
"addition_date": "2017/08/16",
|
|
@@ -2280,8 +2056,7 @@
|
|
|
2280
2056
|
"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"
|
|
2281
2057
|
],
|
|
2282
2058
|
"url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"
|
|
2283
|
-
}
|
|
2284
|
-
,
|
|
2059
|
+
},
|
|
2285
2060
|
{
|
|
2286
2061
|
"pattern": "Jugendschutzprogramm-Crawler",
|
|
2287
2062
|
"addition_date": "2017/08/16",
|
|
@@ -2289,8 +2064,7 @@
|
|
|
2289
2064
|
"Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"
|
|
2290
2065
|
],
|
|
2291
2066
|
"url": "http://www.jugendschutzprogramm.de"
|
|
2292
|
-
}
|
|
2293
|
-
,
|
|
2067
|
+
},
|
|
2294
2068
|
{
|
|
2295
2069
|
"pattern": "Xenu Link Sleuth",
|
|
2296
2070
|
"addition_date": "2017/08/19",
|
|
@@ -2298,8 +2072,7 @@
|
|
|
2298
2072
|
"Xenu Link Sleuth/1.3.8"
|
|
2299
2073
|
],
|
|
2300
2074
|
"url": "http://home.snafu.de/tilman/xenulink.html"
|
|
2301
|
-
}
|
|
2302
|
-
,
|
|
2075
|
+
},
|
|
2303
2076
|
{
|
|
2304
2077
|
"pattern": "Pcore-HTTP",
|
|
2305
2078
|
"addition_date": "2017/08/19",
|
|
@@ -2308,8 +2081,7 @@
|
|
|
2308
2081
|
"Pcore-HTTP/v0.44.0"
|
|
2309
2082
|
],
|
|
2310
2083
|
"url": "https://bitbucket.org/softvisio/pcore/overview"
|
|
2311
|
-
}
|
|
2312
|
-
,
|
|
2084
|
+
},
|
|
2313
2085
|
{
|
|
2314
2086
|
"pattern": "moatbot",
|
|
2315
2087
|
"addition_date": "2017/09/16",
|
|
@@ -2318,8 +2090,7 @@
|
|
|
2318
2090
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"
|
|
2319
2091
|
],
|
|
2320
2092
|
"url": "https://moat.com"
|
|
2321
|
-
}
|
|
2322
|
-
,
|
|
2093
|
+
},
|
|
2323
2094
|
{
|
|
2324
2095
|
"pattern": "KosmioBot",
|
|
2325
2096
|
"addition_date": "2017/09/16",
|
|
@@ -2327,8 +2098,7 @@
|
|
|
2327
2098
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"
|
|
2328
2099
|
],
|
|
2329
2100
|
"url": "http://kosm.io/bot.html"
|
|
2330
|
-
}
|
|
2331
|
-
,
|
|
2101
|
+
},
|
|
2332
2102
|
{
|
|
2333
2103
|
"pattern": "[pP]ingdom",
|
|
2334
2104
|
"addition_date": "2017/09/16",
|
|
@@ -2344,8 +2114,7 @@
|
|
|
2344
2114
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)"
|
|
2345
2115
|
],
|
|
2346
2116
|
"url": "http://www.pingdom.com"
|
|
2347
|
-
}
|
|
2348
|
-
,
|
|
2117
|
+
},
|
|
2349
2118
|
{
|
|
2350
2119
|
"pattern": "AppInsights",
|
|
2351
2120
|
"addition_date": "2019/03/09",
|
|
@@ -2353,8 +2122,7 @@
|
|
|
2353
2122
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)"
|
|
2354
2123
|
],
|
|
2355
2124
|
"url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview"
|
|
2356
|
-
}
|
|
2357
|
-
,
|
|
2125
|
+
},
|
|
2358
2126
|
{
|
|
2359
2127
|
"pattern": "PhantomJS",
|
|
2360
2128
|
"addition_date": "2017/09/18",
|
|
@@ -2362,8 +2130,7 @@
|
|
|
2362
2130
|
"Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"
|
|
2363
2131
|
],
|
|
2364
2132
|
"url": "http://phantomjs.org/"
|
|
2365
|
-
}
|
|
2366
|
-
,
|
|
2133
|
+
},
|
|
2367
2134
|
{
|
|
2368
2135
|
"pattern": "Gowikibot",
|
|
2369
2136
|
"addition_date": "2017/10/26",
|
|
@@ -2371,8 +2138,7 @@
|
|
|
2371
2138
|
"Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"
|
|
2372
2139
|
],
|
|
2373
2140
|
"url": "http://www.gowikibot.com"
|
|
2374
|
-
}
|
|
2375
|
-
,
|
|
2141
|
+
},
|
|
2376
2142
|
{
|
|
2377
2143
|
"pattern": "PiplBot",
|
|
2378
2144
|
"addition_date": "2017/10/30",
|
|
@@ -2381,8 +2147,7 @@
|
|
|
2381
2147
|
"Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"
|
|
2382
2148
|
],
|
|
2383
2149
|
"url": "http://www.pipl.com/bot/"
|
|
2384
|
-
}
|
|
2385
|
-
,
|
|
2150
|
+
},
|
|
2386
2151
|
{
|
|
2387
2152
|
"pattern": "Discordbot",
|
|
2388
2153
|
"addition_date": "2017/09/22",
|
|
@@ -2390,16 +2155,14 @@
|
|
|
2390
2155
|
"instances": [
|
|
2391
2156
|
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
|
|
2392
2157
|
]
|
|
2393
|
-
}
|
|
2394
|
-
,
|
|
2158
|
+
},
|
|
2395
2159
|
{
|
|
2396
2160
|
"pattern": "TelegramBot",
|
|
2397
2161
|
"addition_date": "2017/10/01",
|
|
2398
2162
|
"instances": [
|
|
2399
2163
|
"TelegramBot (like TwitterBot)"
|
|
2400
2164
|
]
|
|
2401
|
-
}
|
|
2402
|
-
,
|
|
2165
|
+
},
|
|
2403
2166
|
{
|
|
2404
2167
|
"pattern": "Jetslide",
|
|
2405
2168
|
"addition_date": "2017/09/27",
|
|
@@ -2407,8 +2170,7 @@
|
|
|
2407
2170
|
"instances": [
|
|
2408
2171
|
"Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"
|
|
2409
2172
|
]
|
|
2410
|
-
}
|
|
2411
|
-
,
|
|
2173
|
+
},
|
|
2412
2174
|
{
|
|
2413
2175
|
"pattern": "newsharecounts",
|
|
2414
2176
|
"addition_date": "2017/09/30",
|
|
@@ -2416,8 +2178,7 @@
|
|
|
2416
2178
|
"instances": [
|
|
2417
2179
|
"Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"
|
|
2418
2180
|
]
|
|
2419
|
-
}
|
|
2420
|
-
,
|
|
2181
|
+
},
|
|
2421
2182
|
{
|
|
2422
2183
|
"pattern": "James BOT",
|
|
2423
2184
|
"addition_date": "2017/10/12",
|
|
@@ -2425,8 +2186,7 @@
|
|
|
2425
2186
|
"instances": [
|
|
2426
2187
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"
|
|
2427
2188
|
]
|
|
2428
|
-
}
|
|
2429
|
-
,
|
|
2189
|
+
},
|
|
2430
2190
|
{
|
|
2431
2191
|
"pattern": "Bark[rR]owler",
|
|
2432
2192
|
"addition_date": "2017/10/09",
|
|
@@ -2437,8 +2197,7 @@
|
|
|
2437
2197
|
"BarkRowler/0.7 (+http://www.exensa.com/crawling)",
|
|
2438
2198
|
"Barkrowler/0.9 (+http://www.exensa.com/crawl)"
|
|
2439
2199
|
]
|
|
2440
|
-
}
|
|
2441
|
-
,
|
|
2200
|
+
},
|
|
2442
2201
|
{
|
|
2443
2202
|
"pattern": "TinEye",
|
|
2444
2203
|
"addition_date": "2017/10/14",
|
|
@@ -2447,8 +2206,7 @@
|
|
|
2447
2206
|
"Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)",
|
|
2448
2207
|
"TinEye/1.1 (http://tineye.com/crawler.html)"
|
|
2449
2208
|
]
|
|
2450
|
-
}
|
|
2451
|
-
,
|
|
2209
|
+
},
|
|
2452
2210
|
{
|
|
2453
2211
|
"pattern": "SocialRankIOBot",
|
|
2454
2212
|
"addition_date": "2017/10/19",
|
|
@@ -2456,8 +2214,7 @@
|
|
|
2456
2214
|
"instances": [
|
|
2457
2215
|
"SocialRankIOBot; http://socialrank.io/about"
|
|
2458
2216
|
]
|
|
2459
|
-
}
|
|
2460
|
-
,
|
|
2217
|
+
},
|
|
2461
2218
|
{
|
|
2462
2219
|
"pattern": "trendictionbot",
|
|
2463
2220
|
"addition_date": "2017/10/30",
|
|
@@ -2466,16 +2223,14 @@
|
|
|
2466
2223
|
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",
|
|
2467
2224
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0"
|
|
2468
2225
|
]
|
|
2469
|
-
}
|
|
2470
|
-
,
|
|
2226
|
+
},
|
|
2471
2227
|
{
|
|
2472
2228
|
"pattern": "Ocarinabot",
|
|
2473
2229
|
"addition_date": "2017/09/27",
|
|
2474
2230
|
"instances": [
|
|
2475
2231
|
"Ocarinabot"
|
|
2476
2232
|
]
|
|
2477
|
-
}
|
|
2478
|
-
,
|
|
2233
|
+
},
|
|
2479
2234
|
{
|
|
2480
2235
|
"pattern": "epicbot",
|
|
2481
2236
|
"addition_date": "2017/10/31",
|
|
@@ -2483,8 +2238,7 @@
|
|
|
2483
2238
|
"instances": [
|
|
2484
2239
|
"Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"
|
|
2485
2240
|
]
|
|
2486
|
-
}
|
|
2487
|
-
,
|
|
2241
|
+
},
|
|
2488
2242
|
{
|
|
2489
2243
|
"pattern": "Primalbot",
|
|
2490
2244
|
"addition_date": "2017/09/27",
|
|
@@ -2492,8 +2246,7 @@
|
|
|
2492
2246
|
"instances": [
|
|
2493
2247
|
"Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"
|
|
2494
2248
|
]
|
|
2495
|
-
}
|
|
2496
|
-
,
|
|
2249
|
+
},
|
|
2497
2250
|
{
|
|
2498
2251
|
"pattern": "DuckDuckGo-Favicons-Bot",
|
|
2499
2252
|
"addition_date": "2017/10/06",
|
|
@@ -2501,8 +2254,7 @@
|
|
|
2501
2254
|
"instances": [
|
|
2502
2255
|
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"
|
|
2503
2256
|
]
|
|
2504
|
-
}
|
|
2505
|
-
,
|
|
2257
|
+
},
|
|
2506
2258
|
{
|
|
2507
2259
|
"pattern": "GnowitNewsbot",
|
|
2508
2260
|
"addition_date": "2017/10/30",
|
|
@@ -2510,8 +2262,7 @@
|
|
|
2510
2262
|
"instances": [
|
|
2511
2263
|
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"
|
|
2512
2264
|
]
|
|
2513
|
-
}
|
|
2514
|
-
,
|
|
2265
|
+
},
|
|
2515
2266
|
{
|
|
2516
2267
|
"pattern": "Leikibot",
|
|
2517
2268
|
"addition_date": "2017/09/24",
|
|
@@ -2519,16 +2270,14 @@
|
|
|
2519
2270
|
"instances": [
|
|
2520
2271
|
"Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"
|
|
2521
2272
|
]
|
|
2522
|
-
}
|
|
2523
|
-
,
|
|
2273
|
+
},
|
|
2524
2274
|
{
|
|
2525
2275
|
"pattern": "LinkArchiver",
|
|
2526
2276
|
"addition_date": "2017/09/24",
|
|
2527
2277
|
"instances": [
|
|
2528
2278
|
"@LinkArchiver twitter bot"
|
|
2529
2279
|
]
|
|
2530
|
-
}
|
|
2531
|
-
,
|
|
2280
|
+
},
|
|
2532
2281
|
{
|
|
2533
2282
|
"pattern": "YaK\\/",
|
|
2534
2283
|
"addition_date": "2017/09/25",
|
|
@@ -2536,8 +2285,7 @@
|
|
|
2536
2285
|
"instances": [
|
|
2537
2286
|
"Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"
|
|
2538
2287
|
]
|
|
2539
|
-
}
|
|
2540
|
-
,
|
|
2288
|
+
},
|
|
2541
2289
|
{
|
|
2542
2290
|
"pattern": "PaperLiBot",
|
|
2543
2291
|
"addition_date": "2017/09/25",
|
|
@@ -2545,10 +2293,8 @@
|
|
|
2545
2293
|
"instances": [
|
|
2546
2294
|
"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",
|
|
2547
2295
|
"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"
|
|
2548
|
-
|
|
2549
2296
|
]
|
|
2550
|
-
}
|
|
2551
|
-
,
|
|
2297
|
+
},
|
|
2552
2298
|
{
|
|
2553
2299
|
"pattern": "Digg Deeper",
|
|
2554
2300
|
"addition_date": "2017/09/26",
|
|
@@ -2556,16 +2302,14 @@
|
|
|
2556
2302
|
"instances": [
|
|
2557
2303
|
"Digg Deeper/v1 (http://digg.com/about)"
|
|
2558
2304
|
]
|
|
2559
|
-
}
|
|
2560
|
-
,
|
|
2305
|
+
},
|
|
2561
2306
|
{
|
|
2562
2307
|
"pattern": "dcrawl",
|
|
2563
2308
|
"addition_date": "2017/09/22",
|
|
2564
2309
|
"instances": [
|
|
2565
2310
|
"dcrawl/1.0"
|
|
2566
2311
|
]
|
|
2567
|
-
}
|
|
2568
|
-
,
|
|
2312
|
+
},
|
|
2569
2313
|
{
|
|
2570
2314
|
"pattern": "Snacktory",
|
|
2571
2315
|
"addition_date": "2017/09/23",
|
|
@@ -2573,8 +2317,7 @@
|
|
|
2573
2317
|
"instances": [
|
|
2574
2318
|
"Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"
|
|
2575
2319
|
]
|
|
2576
|
-
}
|
|
2577
|
-
,
|
|
2320
|
+
},
|
|
2578
2321
|
{
|
|
2579
2322
|
"pattern": "AndersPinkBot",
|
|
2580
2323
|
"addition_date": "2017/09/24",
|
|
@@ -2582,16 +2325,14 @@
|
|
|
2582
2325
|
"instances": [
|
|
2583
2326
|
"Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"
|
|
2584
2327
|
]
|
|
2585
|
-
}
|
|
2586
|
-
,
|
|
2328
|
+
},
|
|
2587
2329
|
{
|
|
2588
2330
|
"pattern": "Fyrebot",
|
|
2589
2331
|
"addition_date": "2017/09/22",
|
|
2590
2332
|
"instances": [
|
|
2591
2333
|
"Fyrebot/1.0"
|
|
2592
2334
|
]
|
|
2593
|
-
}
|
|
2594
|
-
,
|
|
2335
|
+
},
|
|
2595
2336
|
{
|
|
2596
2337
|
"pattern": "EveryoneSocialBot",
|
|
2597
2338
|
"addition_date": "2017/09/22",
|
|
@@ -2599,8 +2340,7 @@
|
|
|
2599
2340
|
"instances": [
|
|
2600
2341
|
"Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"
|
|
2601
2342
|
]
|
|
2602
|
-
}
|
|
2603
|
-
,
|
|
2343
|
+
},
|
|
2604
2344
|
{
|
|
2605
2345
|
"pattern": "Mediatoolkitbot",
|
|
2606
2346
|
"addition_date": "2017/10/06",
|
|
@@ -2608,16 +2348,14 @@
|
|
|
2608
2348
|
"instances": [
|
|
2609
2349
|
"Mediatoolkitbot (complaints@mediatoolkit.com)"
|
|
2610
2350
|
]
|
|
2611
|
-
}
|
|
2612
|
-
,
|
|
2351
|
+
},
|
|
2613
2352
|
{
|
|
2614
2353
|
"pattern": "Luminator-robots",
|
|
2615
2354
|
"addition_date": "2017/09/22",
|
|
2616
2355
|
"instances": [
|
|
2617
2356
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"
|
|
2618
2357
|
]
|
|
2619
|
-
}
|
|
2620
|
-
,
|
|
2358
|
+
},
|
|
2621
2359
|
{
|
|
2622
2360
|
"pattern": "ExtLinksBot",
|
|
2623
2361
|
"addition_date": "2017/11/02",
|
|
@@ -2625,24 +2363,21 @@
|
|
|
2625
2363
|
"instances": [
|
|
2626
2364
|
"Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"
|
|
2627
2365
|
]
|
|
2628
|
-
}
|
|
2629
|
-
,
|
|
2366
|
+
},
|
|
2630
2367
|
{
|
|
2631
2368
|
"pattern": "SurveyBot",
|
|
2632
2369
|
"addition_date": "2017/11/02",
|
|
2633
2370
|
"instances": [
|
|
2634
2371
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"
|
|
2635
2372
|
]
|
|
2636
|
-
}
|
|
2637
|
-
,
|
|
2373
|
+
},
|
|
2638
2374
|
{
|
|
2639
2375
|
"pattern": "NING\\/",
|
|
2640
2376
|
"addition_date": "2017/11/02",
|
|
2641
2377
|
"instances": [
|
|
2642
2378
|
"NING/1.0"
|
|
2643
2379
|
]
|
|
2644
|
-
}
|
|
2645
|
-
,
|
|
2380
|
+
},
|
|
2646
2381
|
{
|
|
2647
2382
|
"pattern": "okhttp",
|
|
2648
2383
|
"addition_date": "2017/11/02",
|
|
@@ -2653,16 +2388,14 @@
|
|
|
2653
2388
|
"okhttp/3.5.0",
|
|
2654
2389
|
"okhttp/4.1.0"
|
|
2655
2390
|
]
|
|
2656
|
-
}
|
|
2657
|
-
,
|
|
2391
|
+
},
|
|
2658
2392
|
{
|
|
2659
2393
|
"pattern": "Nuzzel",
|
|
2660
2394
|
"addition_date": "2017/11/02",
|
|
2661
2395
|
"instances": [
|
|
2662
2396
|
"Nuzzel"
|
|
2663
2397
|
]
|
|
2664
|
-
}
|
|
2665
|
-
,
|
|
2398
|
+
},
|
|
2666
2399
|
{
|
|
2667
2400
|
"pattern": "omgili",
|
|
2668
2401
|
"addition_date": "2017/11/02",
|
|
@@ -2670,8 +2403,7 @@
|
|
|
2670
2403
|
"instances": [
|
|
2671
2404
|
"omgili/0.5 +http://omgili.com"
|
|
2672
2405
|
]
|
|
2673
|
-
}
|
|
2674
|
-
,
|
|
2406
|
+
},
|
|
2675
2407
|
{
|
|
2676
2408
|
"pattern": "PocketParser",
|
|
2677
2409
|
"addition_date": "2017/11/02",
|
|
@@ -2679,8 +2411,7 @@
|
|
|
2679
2411
|
"instances": [
|
|
2680
2412
|
"PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"
|
|
2681
2413
|
]
|
|
2682
|
-
}
|
|
2683
|
-
,
|
|
2414
|
+
},
|
|
2684
2415
|
{
|
|
2685
2416
|
"pattern": "YisouSpider",
|
|
2686
2417
|
"addition_date": "2017/11/02",
|
|
@@ -2688,16 +2419,14 @@
|
|
|
2688
2419
|
"YisouSpider",
|
|
2689
2420
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36"
|
|
2690
2421
|
]
|
|
2691
|
-
}
|
|
2692
|
-
,
|
|
2422
|
+
},
|
|
2693
2423
|
{
|
|
2694
2424
|
"pattern": "um-LN",
|
|
2695
2425
|
"addition_date": "2017/11/02",
|
|
2696
2426
|
"instances": [
|
|
2697
2427
|
"Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"
|
|
2698
2428
|
]
|
|
2699
|
-
}
|
|
2700
|
-
,
|
|
2429
|
+
},
|
|
2701
2430
|
{
|
|
2702
2431
|
"pattern": "ToutiaoSpider",
|
|
2703
2432
|
"addition_date": "2017/11/02",
|
|
@@ -2705,8 +2434,7 @@
|
|
|
2705
2434
|
"instances": [
|
|
2706
2435
|
"Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"
|
|
2707
2436
|
]
|
|
2708
|
-
}
|
|
2709
|
-
,
|
|
2437
|
+
},
|
|
2710
2438
|
{
|
|
2711
2439
|
"pattern": "MuckRack",
|
|
2712
2440
|
"addition_date": "2017/11/02",
|
|
@@ -2714,8 +2442,7 @@
|
|
|
2714
2442
|
"instances": [
|
|
2715
2443
|
"Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"
|
|
2716
2444
|
]
|
|
2717
|
-
}
|
|
2718
|
-
,
|
|
2445
|
+
},
|
|
2719
2446
|
{
|
|
2720
2447
|
"pattern": "Jamie's Spider",
|
|
2721
2448
|
"addition_date": "2017/11/02",
|
|
@@ -2723,32 +2450,28 @@
|
|
|
2723
2450
|
"instances": [
|
|
2724
2451
|
"Jamie's Spider (http://jamiembrown.com/)"
|
|
2725
2452
|
]
|
|
2726
|
-
}
|
|
2727
|
-
,
|
|
2453
|
+
},
|
|
2728
2454
|
{
|
|
2729
2455
|
"pattern": "AHC\\/",
|
|
2730
2456
|
"addition_date": "2017/11/02",
|
|
2731
2457
|
"instances": [
|
|
2732
2458
|
"AHC/2.0"
|
|
2733
2459
|
]
|
|
2734
|
-
}
|
|
2735
|
-
,
|
|
2460
|
+
},
|
|
2736
2461
|
{
|
|
2737
2462
|
"pattern": "NetcraftSurveyAgent",
|
|
2738
2463
|
"addition_date": "2017/11/02",
|
|
2739
2464
|
"instances": [
|
|
2740
2465
|
"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"
|
|
2741
2466
|
]
|
|
2742
|
-
}
|
|
2743
|
-
,
|
|
2467
|
+
},
|
|
2744
2468
|
{
|
|
2745
2469
|
"pattern": "Laserlikebot",
|
|
2746
2470
|
"addition_date": "2017/11/02",
|
|
2747
2471
|
"instances": [
|
|
2748
2472
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"
|
|
2749
2473
|
]
|
|
2750
|
-
}
|
|
2751
|
-
,
|
|
2474
|
+
},
|
|
2752
2475
|
{
|
|
2753
2476
|
"pattern": "^Apache-HttpClient",
|
|
2754
2477
|
"addition_date": "2017/11/02",
|
|
@@ -2768,8 +2491,7 @@
|
|
|
2768
2491
|
"Apache-HttpClient/4.5.7 (Java/11.0.3)",
|
|
2769
2492
|
"Apache-HttpClient/4.5.10 (Java/1.8.0_201)"
|
|
2770
2493
|
]
|
|
2771
|
-
}
|
|
2772
|
-
,
|
|
2494
|
+
},
|
|
2773
2495
|
{
|
|
2774
2496
|
"pattern": "AppEngine-Google",
|
|
2775
2497
|
"addition_date": "2017/11/02",
|
|
@@ -2777,24 +2499,21 @@
|
|
|
2777
2499
|
"AppEngine-Google; (+http://code.google.com/appengine; appid: example)",
|
|
2778
2500
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)"
|
|
2779
2501
|
]
|
|
2780
|
-
}
|
|
2781
|
-
,
|
|
2502
|
+
},
|
|
2782
2503
|
{
|
|
2783
2504
|
"pattern": "Jetty",
|
|
2784
2505
|
"addition_date": "2017/11/02",
|
|
2785
2506
|
"instances": [
|
|
2786
2507
|
"Jetty/9.3.z-SNAPSHOT"
|
|
2787
2508
|
]
|
|
2788
|
-
}
|
|
2789
|
-
,
|
|
2509
|
+
},
|
|
2790
2510
|
{
|
|
2791
2511
|
"pattern": "Upflow",
|
|
2792
2512
|
"addition_date": "2017/11/02",
|
|
2793
2513
|
"instances": [
|
|
2794
2514
|
"Upflow/1.0"
|
|
2795
2515
|
]
|
|
2796
|
-
}
|
|
2797
|
-
,
|
|
2516
|
+
},
|
|
2798
2517
|
{
|
|
2799
2518
|
"pattern": "Thinklab",
|
|
2800
2519
|
"addition_date": "2017/11/02",
|
|
@@ -2802,8 +2521,7 @@
|
|
|
2802
2521
|
"instances": [
|
|
2803
2522
|
"Thinklab (thinklab.com)"
|
|
2804
2523
|
]
|
|
2805
|
-
}
|
|
2806
|
-
,
|
|
2524
|
+
},
|
|
2807
2525
|
{
|
|
2808
2526
|
"pattern": "Traackr\\.com",
|
|
2809
2527
|
"addition_date": "2017/11/02",
|
|
@@ -2811,8 +2529,7 @@
|
|
|
2811
2529
|
"instances": [
|
|
2812
2530
|
"Traackr.com"
|
|
2813
2531
|
]
|
|
2814
|
-
}
|
|
2815
|
-
,
|
|
2532
|
+
},
|
|
2816
2533
|
{
|
|
2817
2534
|
"pattern": "Twurly",
|
|
2818
2535
|
"addition_date": "2017/11/02",
|
|
@@ -2820,56 +2537,49 @@
|
|
|
2820
2537
|
"instances": [
|
|
2821
2538
|
"Ruby, Twurly v1.1 (http://twurly.org)"
|
|
2822
2539
|
]
|
|
2823
|
-
}
|
|
2824
|
-
,
|
|
2540
|
+
},
|
|
2825
2541
|
{
|
|
2826
2542
|
"pattern": "Mastodon",
|
|
2827
2543
|
"addition_date": "2017/11/02",
|
|
2828
2544
|
"instances": [
|
|
2829
2545
|
"http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"
|
|
2830
2546
|
]
|
|
2831
|
-
}
|
|
2832
|
-
,
|
|
2547
|
+
},
|
|
2833
2548
|
{
|
|
2834
2549
|
"pattern": "http_get",
|
|
2835
2550
|
"addition_date": "2017/11/02",
|
|
2836
2551
|
"instances": [
|
|
2837
2552
|
"http_get"
|
|
2838
2553
|
]
|
|
2839
|
-
}
|
|
2840
|
-
,
|
|
2554
|
+
},
|
|
2841
2555
|
{
|
|
2842
2556
|
"pattern": "DnyzBot",
|
|
2843
2557
|
"addition_date": "2017/11/20",
|
|
2844
2558
|
"instances": [
|
|
2845
2559
|
"Mozilla/5.0 (compatible; DnyzBot/1.0)"
|
|
2846
2560
|
]
|
|
2847
|
-
}
|
|
2848
|
-
,
|
|
2561
|
+
},
|
|
2849
2562
|
{
|
|
2850
2563
|
"pattern": "botify",
|
|
2851
2564
|
"addition_date": "2018/02/01",
|
|
2852
2565
|
"instances": [
|
|
2853
2566
|
"Mozilla/5.0 (compatible; botify; http://botify.com)"
|
|
2854
2567
|
]
|
|
2855
|
-
}
|
|
2856
|
-
,
|
|
2568
|
+
},
|
|
2857
2569
|
{
|
|
2858
2570
|
"pattern": "007ac9 Crawler",
|
|
2859
2571
|
"addition_date": "2018/02/09",
|
|
2860
2572
|
"instances": [
|
|
2861
2573
|
"Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)"
|
|
2862
2574
|
]
|
|
2863
|
-
}
|
|
2864
|
-
,
|
|
2575
|
+
},
|
|
2865
2576
|
{
|
|
2866
2577
|
"pattern": "BehloolBot",
|
|
2867
2578
|
"addition_date": "2018/02/09",
|
|
2868
2579
|
"instances": [
|
|
2869
2580
|
"Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)"
|
|
2870
2581
|
]
|
|
2871
|
-
}
|
|
2872
|
-
,
|
|
2582
|
+
},
|
|
2873
2583
|
{
|
|
2874
2584
|
"pattern": "BrandVerity",
|
|
2875
2585
|
"addition_date": "2018/02/27",
|
|
@@ -2878,16 +2588,14 @@
|
|
|
2878
2588
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)"
|
|
2879
2589
|
],
|
|
2880
2590
|
"url": "http://www.brandverity.com/why-is-brandverity-visiting-me"
|
|
2881
|
-
}
|
|
2882
|
-
,
|
|
2591
|
+
},
|
|
2883
2592
|
{
|
|
2884
2593
|
"pattern": "check_http",
|
|
2885
2594
|
"addition_date": "2018/02/09",
|
|
2886
2595
|
"instances": [
|
|
2887
2596
|
"check_http/v2.2.1 (nagios-plugins 2.2.1)"
|
|
2888
2597
|
]
|
|
2889
|
-
}
|
|
2890
|
-
,
|
|
2598
|
+
},
|
|
2891
2599
|
{
|
|
2892
2600
|
"pattern": "BDCbot",
|
|
2893
2601
|
"addition_date": "2018/02/09",
|
|
@@ -2895,24 +2603,21 @@
|
|
|
2895
2603
|
"Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36",
|
|
2896
2604
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
|
|
2897
2605
|
]
|
|
2898
|
-
}
|
|
2899
|
-
,
|
|
2606
|
+
},
|
|
2900
2607
|
{
|
|
2901
2608
|
"pattern": "ZumBot",
|
|
2902
2609
|
"addition_date": "2018/02/09",
|
|
2903
2610
|
"instances": [
|
|
2904
2611
|
"Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)"
|
|
2905
2612
|
]
|
|
2906
|
-
}
|
|
2907
|
-
,
|
|
2613
|
+
},
|
|
2908
2614
|
{
|
|
2909
2615
|
"pattern": "EZID",
|
|
2910
2616
|
"addition_date": "2018/02/09",
|
|
2911
2617
|
"instances": [
|
|
2912
2618
|
"EZID (EZID link checker; https://ezid.cdlib.org/)"
|
|
2913
2619
|
]
|
|
2914
|
-
}
|
|
2915
|
-
,
|
|
2620
|
+
},
|
|
2916
2621
|
{
|
|
2917
2622
|
"pattern": "ICC-Crawler",
|
|
2918
2623
|
"addition_date": "2018/02/28",
|
|
@@ -2920,8 +2625,7 @@
|
|
|
2920
2625
|
"ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)"
|
|
2921
2626
|
],
|
|
2922
2627
|
"url": "http://ucri.nict.go.jp/en/icccrawler.html"
|
|
2923
|
-
}
|
|
2924
|
-
,
|
|
2628
|
+
},
|
|
2925
2629
|
{
|
|
2926
2630
|
"pattern": "ArchiveBot",
|
|
2927
2631
|
"addition_date": "2018/02/28",
|
|
@@ -2929,8 +2633,7 @@
|
|
|
2929
2633
|
"ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)"
|
|
2930
2634
|
],
|
|
2931
2635
|
"url": "https://github.com/ArchiveTeam/ArchiveBot"
|
|
2932
|
-
}
|
|
2933
|
-
,
|
|
2636
|
+
},
|
|
2934
2637
|
{
|
|
2935
2638
|
"pattern": "^LCC ",
|
|
2936
2639
|
"addition_date": "2018/02/28",
|
|
@@ -2938,8 +2641,7 @@
|
|
|
2938
2641
|
"LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)"
|
|
2939
2642
|
],
|
|
2940
2643
|
"url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html"
|
|
2941
|
-
}
|
|
2942
|
-
,
|
|
2644
|
+
},
|
|
2943
2645
|
{
|
|
2944
2646
|
"pattern": "filterdb\\.iss\\.net\\/crawler",
|
|
2945
2647
|
"addition_date": "2018/03/16",
|
|
@@ -2947,16 +2649,14 @@
|
|
|
2947
2649
|
"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)"
|
|
2948
2650
|
],
|
|
2949
2651
|
"url": "http://filterdb.iss.net/crawler/"
|
|
2950
|
-
}
|
|
2951
|
-
,
|
|
2652
|
+
},
|
|
2952
2653
|
{
|
|
2953
2654
|
"pattern": "BLP_bbot",
|
|
2954
2655
|
"addition_date": "2018/03/27",
|
|
2955
2656
|
"instances": [
|
|
2956
2657
|
"BLP_bbot/0.1"
|
|
2957
2658
|
]
|
|
2958
|
-
}
|
|
2959
|
-
,
|
|
2659
|
+
},
|
|
2960
2660
|
{
|
|
2961
2661
|
"pattern": "BomboraBot",
|
|
2962
2662
|
"addition_date": "2018/03/27",
|
|
@@ -2964,8 +2664,7 @@
|
|
|
2964
2664
|
"Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)"
|
|
2965
2665
|
],
|
|
2966
2666
|
"url": "http://www.bombora.com/bot"
|
|
2967
|
-
}
|
|
2968
|
-
,
|
|
2667
|
+
},
|
|
2969
2668
|
{
|
|
2970
2669
|
"pattern": "Buck\\/",
|
|
2971
2670
|
"addition_date": "2018/03/27",
|
|
@@ -2973,8 +2672,7 @@
|
|
|
2973
2672
|
"Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)"
|
|
2974
2673
|
],
|
|
2975
2674
|
"url": "https://app.hypefactors.com/media-monitoring/about.html"
|
|
2976
|
-
}
|
|
2977
|
-
,
|
|
2675
|
+
},
|
|
2978
2676
|
{
|
|
2979
2677
|
"pattern": "Companybook-Crawler",
|
|
2980
2678
|
"addition_date": "2018/03/27",
|
|
@@ -2982,8 +2680,7 @@
|
|
|
2982
2680
|
"Companybook-Crawler (+https://www.companybooknetworking.com/)"
|
|
2983
2681
|
],
|
|
2984
2682
|
"url": "https://www.companybooknetworking.com/"
|
|
2985
|
-
}
|
|
2986
|
-
,
|
|
2683
|
+
},
|
|
2987
2684
|
{
|
|
2988
2685
|
"pattern": "Genieo",
|
|
2989
2686
|
"addition_date": "2018/03/27",
|
|
@@ -2991,8 +2688,7 @@
|
|
|
2991
2688
|
"Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)"
|
|
2992
2689
|
],
|
|
2993
2690
|
"url": "http://www.genieo.com/webfilter.html"
|
|
2994
|
-
}
|
|
2995
|
-
,
|
|
2691
|
+
},
|
|
2996
2692
|
{
|
|
2997
2693
|
"pattern": "magpie-crawler",
|
|
2998
2694
|
"addition_date": "2018/03/27",
|
|
@@ -3000,8 +2696,7 @@
|
|
|
3000
2696
|
"magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)"
|
|
3001
2697
|
],
|
|
3002
2698
|
"url": "http://www.brandwatch.net"
|
|
3003
|
-
}
|
|
3004
|
-
,
|
|
2699
|
+
},
|
|
3005
2700
|
{
|
|
3006
2701
|
"pattern": "MeltwaterNews",
|
|
3007
2702
|
"addition_date": "2018/03/27",
|
|
@@ -3009,8 +2704,7 @@
|
|
|
3009
2704
|
"MeltwaterNews www.meltwater.com"
|
|
3010
2705
|
],
|
|
3011
2706
|
"url": "http://www.meltwater.com"
|
|
3012
|
-
}
|
|
3013
|
-
,
|
|
2707
|
+
},
|
|
3014
2708
|
{
|
|
3015
2709
|
"pattern": "Moreover",
|
|
3016
2710
|
"addition_date": "2018/03/27",
|
|
@@ -3018,8 +2712,7 @@
|
|
|
3018
2712
|
"Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)"
|
|
3019
2713
|
],
|
|
3020
2714
|
"url": "http://www.moreover.com"
|
|
3021
|
-
}
|
|
3022
|
-
,
|
|
2715
|
+
},
|
|
3023
2716
|
{
|
|
3024
2717
|
"pattern": "newspaper\\/",
|
|
3025
2718
|
"addition_date": "2018/03/27",
|
|
@@ -3029,8 +2722,7 @@
|
|
|
3029
2722
|
"newspaper/0.2.6",
|
|
3030
2723
|
"newspaper/0.2.8"
|
|
3031
2724
|
]
|
|
3032
|
-
}
|
|
3033
|
-
,
|
|
2725
|
+
},
|
|
3034
2726
|
{
|
|
3035
2727
|
"pattern": "ScoutJet",
|
|
3036
2728
|
"addition_date": "2018/03/27",
|
|
@@ -3038,8 +2730,7 @@
|
|
|
3038
2730
|
"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)"
|
|
3039
2731
|
],
|
|
3040
2732
|
"url": "http://www.scoutjet.com/"
|
|
3041
|
-
}
|
|
3042
|
-
,
|
|
2733
|
+
},
|
|
3043
2734
|
{
|
|
3044
2735
|
"pattern": "(^| )sentry\\/",
|
|
3045
2736
|
"addition_date": "2018/03/27",
|
|
@@ -3047,8 +2738,7 @@
|
|
|
3047
2738
|
"sentry/8.22.0 (https://sentry.io)"
|
|
3048
2739
|
],
|
|
3049
2740
|
"url": "https://sentry.io"
|
|
3050
|
-
}
|
|
3051
|
-
,
|
|
2741
|
+
},
|
|
3052
2742
|
{
|
|
3053
2743
|
"pattern": "StorygizeBot",
|
|
3054
2744
|
"addition_date": "2018/03/27",
|
|
@@ -3056,8 +2746,7 @@
|
|
|
3056
2746
|
"Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)"
|
|
3057
2747
|
],
|
|
3058
2748
|
"url": "http://www.storygize.com"
|
|
3059
|
-
}
|
|
3060
|
-
,
|
|
2749
|
+
},
|
|
3061
2750
|
{
|
|
3062
2751
|
"pattern": "UptimeRobot",
|
|
3063
2752
|
"addition_date": "2018/03/27",
|
|
@@ -3065,8 +2754,7 @@
|
|
|
3065
2754
|
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"
|
|
3066
2755
|
],
|
|
3067
2756
|
"url": "http://www.uptimerobot.com/"
|
|
3068
|
-
}
|
|
3069
|
-
,
|
|
2757
|
+
},
|
|
3070
2758
|
{
|
|
3071
2759
|
"pattern": "OutclicksBot",
|
|
3072
2760
|
"addition_date": "2018/04/21",
|
|
@@ -3077,8 +2765,7 @@
|
|
|
3077
2765
|
"OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h"
|
|
3078
2766
|
],
|
|
3079
2767
|
"url": "https://www.outclicks.net"
|
|
3080
|
-
}
|
|
3081
|
-
,
|
|
2768
|
+
},
|
|
3082
2769
|
{
|
|
3083
2770
|
"pattern": "seoscanners",
|
|
3084
2771
|
"addition_date": "2018/05/27",
|
|
@@ -3086,8 +2773,7 @@
|
|
|
3086
2773
|
"Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)"
|
|
3087
2774
|
],
|
|
3088
2775
|
"url": "http://www.seoscanners.net/"
|
|
3089
|
-
}
|
|
3090
|
-
,
|
|
2776
|
+
},
|
|
3091
2777
|
{
|
|
3092
2778
|
"pattern": "Hatena",
|
|
3093
2779
|
"addition_date": "2018/05/29",
|
|
@@ -3099,8 +2785,7 @@
|
|
|
3099
2785
|
"HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)",
|
|
3100
2786
|
"Hatena::Fetcher/0.01 (master) Furl/3.13"
|
|
3101
2787
|
]
|
|
3102
|
-
}
|
|
3103
|
-
,
|
|
2788
|
+
},
|
|
3104
2789
|
{
|
|
3105
2790
|
"pattern": "Google Web Preview",
|
|
3106
2791
|
"addition_date": "2018/05/31",
|
|
@@ -3108,16 +2793,14 @@
|
|
|
3108
2793
|
"Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36",
|
|
3109
2794
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36"
|
|
3110
2795
|
]
|
|
3111
|
-
}
|
|
3112
|
-
,
|
|
2796
|
+
},
|
|
3113
2797
|
{
|
|
3114
2798
|
"pattern": "MauiBot",
|
|
3115
2799
|
"addition_date": "2018/06/06",
|
|
3116
2800
|
"instances": [
|
|
3117
2801
|
"MauiBot (crawler.feedback+wc@gmail.com)"
|
|
3118
2802
|
]
|
|
3119
|
-
}
|
|
3120
|
-
,
|
|
2803
|
+
},
|
|
3121
2804
|
{
|
|
3122
2805
|
"pattern": "AlphaBot",
|
|
3123
2806
|
"addition_date": "2018/05/27",
|
|
@@ -3125,8 +2808,7 @@
|
|
|
3125
2808
|
"Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)"
|
|
3126
2809
|
],
|
|
3127
2810
|
"url": "http://alphaseobot.com/bot.html"
|
|
3128
|
-
}
|
|
3129
|
-
,
|
|
2811
|
+
},
|
|
3130
2812
|
{
|
|
3131
2813
|
"pattern": "SBL-BOT",
|
|
3132
2814
|
"addition_date": "2018/06/06",
|
|
@@ -3134,9 +2816,8 @@
|
|
|
3134
2816
|
"SBL-BOT (http://sbl.net)"
|
|
3135
2817
|
],
|
|
3136
2818
|
"url": "http://sbl.net",
|
|
3137
|
-
"description"
|
|
3138
|
-
}
|
|
3139
|
-
,
|
|
2819
|
+
"description": "Bot of SoftByte BlackWidow"
|
|
2820
|
+
},
|
|
3140
2821
|
{
|
|
3141
2822
|
"pattern": "IAS crawler",
|
|
3142
2823
|
"addition_date": "2018/06/06",
|
|
@@ -3144,17 +2825,15 @@
|
|
|
3144
2825
|
"IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)"
|
|
3145
2826
|
],
|
|
3146
2827
|
"url": "http://integralads.com/site-indexing-policy/",
|
|
3147
|
-
"description"
|
|
3148
|
-
}
|
|
3149
|
-
,
|
|
2828
|
+
"description": "Bot of Integral Ad Science, Inc."
|
|
2829
|
+
},
|
|
3150
2830
|
{
|
|
3151
2831
|
"pattern": "adscanner",
|
|
3152
2832
|
"addition_date": "2018/06/24",
|
|
3153
2833
|
"instances": [
|
|
3154
2834
|
"Mozilla/5.0 (compatible; adscanner/)"
|
|
3155
2835
|
]
|
|
3156
|
-
}
|
|
3157
|
-
,
|
|
2836
|
+
},
|
|
3158
2837
|
{
|
|
3159
2838
|
"pattern": "Netvibes",
|
|
3160
2839
|
"addition_date": "2018/06/24",
|
|
@@ -3163,8 +2842,7 @@
|
|
|
3163
2842
|
"Netvibes (crawler; http://www.netvibes.com)"
|
|
3164
2843
|
],
|
|
3165
2844
|
"url": "http://www.netvibes.com"
|
|
3166
|
-
}
|
|
3167
|
-
,
|
|
2845
|
+
},
|
|
3168
2846
|
{
|
|
3169
2847
|
"pattern": "acapbot",
|
|
3170
2848
|
"addition_date": "2018/06/27",
|
|
@@ -3172,8 +2850,7 @@
|
|
|
3172
2850
|
"Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",
|
|
3173
2851
|
"Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)"
|
|
3174
2852
|
]
|
|
3175
|
-
}
|
|
3176
|
-
,
|
|
2853
|
+
},
|
|
3177
2854
|
{
|
|
3178
2855
|
"pattern": "Baidu-YunGuanCe",
|
|
3179
2856
|
"addition_date": "2018/06/27",
|
|
@@ -3186,8 +2863,7 @@
|
|
|
3186
2863
|
],
|
|
3187
2864
|
"url": "https://ce.baidu.com/topic/topic20150908",
|
|
3188
2865
|
"description": "Baidu Cloud Watch"
|
|
3189
|
-
}
|
|
3190
|
-
,
|
|
2866
|
+
},
|
|
3191
2867
|
{
|
|
3192
2868
|
"pattern": "bitlybot",
|
|
3193
2869
|
"addition_date": "2018/06/27",
|
|
@@ -3197,8 +2873,7 @@
|
|
|
3197
2873
|
"bitlybot"
|
|
3198
2874
|
],
|
|
3199
2875
|
"url": "http://bit.ly/"
|
|
3200
|
-
}
|
|
3201
|
-
,
|
|
2876
|
+
},
|
|
3202
2877
|
{
|
|
3203
2878
|
"pattern": "blogmuraBot",
|
|
3204
2879
|
"addition_date": "2018/06/27",
|
|
@@ -3207,8 +2882,7 @@
|
|
|
3207
2882
|
],
|
|
3208
2883
|
"url": "http://www.blogmura.com",
|
|
3209
2884
|
"description": "A blog ranking site which links to blogs on just about every theme possible."
|
|
3210
|
-
}
|
|
3211
|
-
,
|
|
2885
|
+
},
|
|
3212
2886
|
{
|
|
3213
2887
|
"pattern": "Bot\\.AraTurka\\.com",
|
|
3214
2888
|
"addition_date": "2018/06/27",
|
|
@@ -3216,16 +2890,14 @@
|
|
|
3216
2890
|
"Bot.AraTurka.com/0.0.1"
|
|
3217
2891
|
],
|
|
3218
2892
|
"url": "http://www.araturka.com"
|
|
3219
|
-
}
|
|
3220
|
-
,
|
|
2893
|
+
},
|
|
3221
2894
|
{
|
|
3222
2895
|
"pattern": "bot-pge\\.chlooe\\.com",
|
|
3223
2896
|
"addition_date": "2018/06/27",
|
|
3224
2897
|
"instances": [
|
|
3225
2898
|
"bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)"
|
|
3226
2899
|
]
|
|
3227
|
-
}
|
|
3228
|
-
,
|
|
2900
|
+
},
|
|
3229
2901
|
{
|
|
3230
2902
|
"pattern": "BoxcarBot",
|
|
3231
2903
|
"addition_date": "2018/06/27",
|
|
@@ -3233,8 +2905,7 @@
|
|
|
3233
2905
|
"Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)"
|
|
3234
2906
|
],
|
|
3235
2907
|
"url": "https://boxcar.io/"
|
|
3236
|
-
}
|
|
3237
|
-
,
|
|
2908
|
+
},
|
|
3238
2909
|
{
|
|
3239
2910
|
"pattern": "BTWebClient",
|
|
3240
2911
|
"addition_date": "2018/06/27",
|
|
@@ -3243,8 +2914,7 @@
|
|
|
3243
2914
|
],
|
|
3244
2915
|
"url": "http://www.utorrent.com/",
|
|
3245
2916
|
"description": "µTorrent BitTorrent Client"
|
|
3246
|
-
}
|
|
3247
|
-
,
|
|
2917
|
+
},
|
|
3248
2918
|
{
|
|
3249
2919
|
"pattern": "ContextAd Bot",
|
|
3250
2920
|
"addition_date": "2018/06/27",
|
|
@@ -3252,8 +2922,7 @@
|
|
|
3252
2922
|
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)",
|
|
3253
2923
|
"ContextAd Bot 1.0"
|
|
3254
2924
|
]
|
|
3255
|
-
}
|
|
3256
|
-
,
|
|
2925
|
+
},
|
|
3257
2926
|
{
|
|
3258
2927
|
"pattern": "Digincore bot",
|
|
3259
2928
|
"addition_date": "2018/06/27",
|
|
@@ -3261,8 +2930,7 @@
|
|
|
3261
2930
|
"Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)"
|
|
3262
2931
|
],
|
|
3263
2932
|
"url": "http://www.digincore.com/crawler.html"
|
|
3264
|
-
}
|
|
3265
|
-
,
|
|
2933
|
+
},
|
|
3266
2934
|
{
|
|
3267
2935
|
"pattern": "Disqus",
|
|
3268
2936
|
"addition_date": "2018/06/27",
|
|
@@ -3271,8 +2939,7 @@
|
|
|
3271
2939
|
],
|
|
3272
2940
|
"url": "https://disqus.com/",
|
|
3273
2941
|
"description": "validate and quality check pages."
|
|
3274
|
-
}
|
|
3275
|
-
,
|
|
2942
|
+
},
|
|
3276
2943
|
{
|
|
3277
2944
|
"pattern": "Feedly",
|
|
3278
2945
|
"addition_date": "2018/06/27",
|
|
@@ -3282,16 +2949,14 @@
|
|
|
3282
2949
|
],
|
|
3283
2950
|
"url": "https://www.feedly.com/fetcher.html",
|
|
3284
2951
|
"description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud."
|
|
3285
|
-
}
|
|
3286
|
-
,
|
|
2952
|
+
},
|
|
3287
2953
|
{
|
|
3288
2954
|
"pattern": "Fetch\\/",
|
|
3289
2955
|
"addition_date": "2018/06/27",
|
|
3290
2956
|
"instances": [
|
|
3291
2957
|
"Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)"
|
|
3292
2958
|
]
|
|
3293
|
-
}
|
|
3294
|
-
,
|
|
2959
|
+
},
|
|
3295
2960
|
{
|
|
3296
2961
|
"pattern": "Fever",
|
|
3297
2962
|
"addition_date": "2018/06/27",
|
|
@@ -3299,16 +2964,14 @@
|
|
|
3299
2964
|
"Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)"
|
|
3300
2965
|
],
|
|
3301
2966
|
"url": "http://feedafever.com"
|
|
3302
|
-
}
|
|
3303
|
-
,
|
|
2967
|
+
},
|
|
3304
2968
|
{
|
|
3305
2969
|
"pattern": "Flamingo_SearchEngine",
|
|
3306
2970
|
"addition_date": "2018/06/27",
|
|
3307
2971
|
"instances": [
|
|
3308
2972
|
"Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)"
|
|
3309
2973
|
]
|
|
3310
|
-
}
|
|
3311
|
-
,
|
|
2974
|
+
},
|
|
3312
2975
|
{
|
|
3313
2976
|
"pattern": "FlipboardProxy",
|
|
3314
2977
|
"addition_date": "2018/06/27",
|
|
@@ -3321,8 +2984,7 @@
|
|
|
3321
2984
|
],
|
|
3322
2985
|
"url": "https://about.flipboard.com/browserproxy/",
|
|
3323
2986
|
"description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application"
|
|
3324
|
-
}
|
|
3325
|
-
,
|
|
2987
|
+
},
|
|
3326
2988
|
{
|
|
3327
2989
|
"pattern": "g2reader-bot",
|
|
3328
2990
|
"addition_date": "2018/06/27",
|
|
@@ -3330,8 +2992,7 @@
|
|
|
3330
2992
|
"g2reader-bot/1.0 (+http://www.g2reader.com/)"
|
|
3331
2993
|
],
|
|
3332
2994
|
"url": "http://www.g2reader.com/"
|
|
3333
|
-
}
|
|
3334
|
-
,
|
|
2995
|
+
},
|
|
3335
2996
|
{
|
|
3336
2997
|
"pattern": "G2 Web Services",
|
|
3337
2998
|
"addition_date": "2019/03/01",
|
|
@@ -3339,8 +3000,7 @@
|
|
|
3339
3000
|
"G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)"
|
|
3340
3001
|
],
|
|
3341
3002
|
"url": "https://www.g2webservices.com/"
|
|
3342
|
-
}
|
|
3343
|
-
,
|
|
3003
|
+
},
|
|
3344
3004
|
{
|
|
3345
3005
|
"pattern": "imrbot",
|
|
3346
3006
|
"addition_date": "2018/06/27",
|
|
@@ -3348,8 +3008,7 @@
|
|
|
3348
3008
|
"Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)"
|
|
3349
3009
|
],
|
|
3350
3010
|
"url": "http://www.mignify.com"
|
|
3351
|
-
}
|
|
3352
|
-
,
|
|
3011
|
+
},
|
|
3353
3012
|
{
|
|
3354
3013
|
"pattern": "K7MLWCBot",
|
|
3355
3014
|
"addition_date": "2018/06/27",
|
|
@@ -3358,8 +3017,7 @@
|
|
|
3358
3017
|
],
|
|
3359
3018
|
"url": "http://www.k7computing.com",
|
|
3360
3019
|
"description": "Virus scanner"
|
|
3361
|
-
}
|
|
3362
|
-
,
|
|
3020
|
+
},
|
|
3363
3021
|
{
|
|
3364
3022
|
"pattern": "Kemvibot",
|
|
3365
3023
|
"addition_date": "2018/06/27",
|
|
@@ -3367,8 +3025,7 @@
|
|
|
3367
3025
|
"Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)"
|
|
3368
3026
|
],
|
|
3369
3027
|
"url": "http://kemvi.com"
|
|
3370
|
-
}
|
|
3371
|
-
,
|
|
3028
|
+
},
|
|
3372
3029
|
{
|
|
3373
3030
|
"pattern": "Landau-Media-Spider",
|
|
3374
3031
|
"addition_date": "2018/06/27",
|
|
@@ -3376,8 +3033,7 @@
|
|
|
3376
3033
|
"Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)"
|
|
3377
3034
|
],
|
|
3378
3035
|
"url": "http://bots.landaumedia.de/bot.html"
|
|
3379
|
-
}
|
|
3380
|
-
,
|
|
3036
|
+
},
|
|
3381
3037
|
{
|
|
3382
3038
|
"pattern": "linkapediabot",
|
|
3383
3039
|
"addition_date": "2018/06/27",
|
|
@@ -3385,8 +3041,7 @@
|
|
|
3385
3041
|
"linkapediabot (+http://www.linkapedia.com)"
|
|
3386
3042
|
],
|
|
3387
3043
|
"url": "http://www.linkapedia.com"
|
|
3388
|
-
}
|
|
3389
|
-
,
|
|
3044
|
+
},
|
|
3390
3045
|
{
|
|
3391
3046
|
"pattern": "vkShare",
|
|
3392
3047
|
"addition_date": "2018/07/02",
|
|
@@ -3394,8 +3049,7 @@
|
|
|
3394
3049
|
"Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)"
|
|
3395
3050
|
],
|
|
3396
3051
|
"url": "http://vk.com/dev/Share"
|
|
3397
|
-
}
|
|
3398
|
-
,
|
|
3052
|
+
},
|
|
3399
3053
|
{
|
|
3400
3054
|
"pattern": "Siteimprove\\.com",
|
|
3401
3055
|
"addition_date": "2018/06/22",
|
|
@@ -3405,121 +3059,109 @@
|
|
|
3405
3059
|
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com",
|
|
3406
3060
|
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com"
|
|
3407
3061
|
]
|
|
3408
|
-
}
|
|
3409
|
-
,
|
|
3410
|
-
{
|
|
3411
|
-
"pattern": "BLEXBot\\/",
|
|
3412
|
-
"addition_date": "2018/07/07",
|
|
3413
|
-
"instances": [
|
|
3414
|
-
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"
|
|
3415
|
-
],
|
|
3416
|
-
"url": "http://webmeup-crawler.com"
|
|
3417
|
-
}
|
|
3418
|
-
,
|
|
3419
|
-
{
|
|
3420
|
-
"pattern": "DareBoost",
|
|
3421
|
-
"addition_date": "2018/07/07",
|
|
3422
|
-
"instances": [
|
|
3423
|
-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost"
|
|
3424
|
-
],
|
|
3425
|
-
"url": "https://www.dareboost.com/",
|
|
3426
|
-
"description": "Bot to test, Analyze and Optimize website"
|
|
3427
|
-
}
|
|
3428
|
-
,
|
|
3062
|
+
},
|
|
3429
3063
|
{
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
,
|
|
3437
|
-
{
|
|
3438
|
-
"pattern": "Miniflux\\/",
|
|
3439
|
-
"addition_date": "2018/07/07",
|
|
3440
|
-
"instances": [
|
|
3441
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)",
|
|
3442
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)",
|
|
3443
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)",
|
|
3444
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)",
|
|
3445
|
-
"Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)",
|
|
3446
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)",
|
|
3447
|
-
"Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)",
|
|
3448
|
-
"Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)",
|
|
3449
|
-
"Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)"
|
|
3450
|
-
],
|
|
3451
|
-
"url": "https://miniflux.net",
|
|
3452
|
-
"description": "Miniflux is a minimalist and opinionated feed reader."
|
|
3453
|
-
}
|
|
3454
|
-
,
|
|
3455
|
-
{
|
|
3456
|
-
"pattern": "Feedspot",
|
|
3457
|
-
"addition_date": "2018/07/07",
|
|
3458
|
-
"instances": [
|
|
3459
|
-
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)",
|
|
3460
|
-
"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"
|
|
3461
|
-
],
|
|
3462
|
-
"url": "http://www.feedspot.com/fs/bot"
|
|
3463
|
-
}
|
|
3464
|
-
,
|
|
3465
|
-
{
|
|
3466
|
-
"pattern": "Diffbot\\/",
|
|
3467
|
-
"addition_date": "2018/07/07",
|
|
3468
|
-
"instances": [
|
|
3469
|
-
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)"
|
|
3470
|
-
],
|
|
3471
|
-
"url": "http://www.diffbot.com"
|
|
3472
|
-
}
|
|
3473
|
-
,
|
|
3474
|
-
{
|
|
3475
|
-
"pattern": "SEOkicks",
|
|
3476
|
-
"addition_date": "2018/08/22",
|
|
3477
|
-
"instances": [
|
|
3478
|
-
"Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)"
|
|
3479
|
-
],
|
|
3480
|
-
"url": "https://www.seokicks.de/robot.html"
|
|
3481
|
-
}
|
|
3482
|
-
,
|
|
3064
|
+
"pattern": "BLEXBot\\/",
|
|
3065
|
+
"addition_date": "2018/07/07",
|
|
3066
|
+
"instances": [
|
|
3067
|
+
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"
|
|
3068
|
+
],
|
|
3069
|
+
"url": "http://webmeup-crawler.com"
|
|
3070
|
+
},
|
|
3483
3071
|
{
|
|
3484
|
-
|
|
3485
|
-
|
|
3486
|
-
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3072
|
+
"pattern": "DareBoost",
|
|
3073
|
+
"addition_date": "2018/07/07",
|
|
3074
|
+
"instances": [
|
|
3075
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost"
|
|
3076
|
+
],
|
|
3077
|
+
"url": "https://www.dareboost.com/",
|
|
3078
|
+
"description": "Bot to test, Analyze and Optimize website"
|
|
3079
|
+
},
|
|
3491
3080
|
{
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
}
|
|
3498
|
-
|
|
3081
|
+
"pattern": "ZuperlistBot\\/",
|
|
3082
|
+
"addition_date": "2018/07/07",
|
|
3083
|
+
"instances": [
|
|
3084
|
+
"Mozilla/5.0 (compatible; ZuperlistBot/1.0)"
|
|
3085
|
+
]
|
|
3086
|
+
},
|
|
3087
|
+
{
|
|
3088
|
+
"pattern": "Miniflux\\/",
|
|
3089
|
+
"addition_date": "2018/07/07",
|
|
3090
|
+
"instances": [
|
|
3091
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)",
|
|
3092
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)",
|
|
3093
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)",
|
|
3094
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)",
|
|
3095
|
+
"Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)",
|
|
3096
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)",
|
|
3097
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)",
|
|
3098
|
+
"Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)",
|
|
3099
|
+
"Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)"
|
|
3100
|
+
],
|
|
3101
|
+
"url": "https://miniflux.net",
|
|
3102
|
+
"description": "Miniflux is a minimalist and opinionated feed reader."
|
|
3103
|
+
},
|
|
3104
|
+
{
|
|
3105
|
+
"pattern": "Feedspot",
|
|
3106
|
+
"addition_date": "2018/07/07",
|
|
3107
|
+
"instances": [
|
|
3108
|
+
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)",
|
|
3109
|
+
"Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"
|
|
3110
|
+
],
|
|
3111
|
+
"url": "http://www.feedspot.com/fs/bot"
|
|
3112
|
+
},
|
|
3113
|
+
{
|
|
3114
|
+
"pattern": "Diffbot\\/",
|
|
3115
|
+
"addition_date": "2018/07/07",
|
|
3116
|
+
"instances": [
|
|
3117
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)"
|
|
3118
|
+
],
|
|
3119
|
+
"url": "http://www.diffbot.com"
|
|
3120
|
+
},
|
|
3121
|
+
{
|
|
3122
|
+
"pattern": "SEOkicks",
|
|
3123
|
+
"addition_date": "2018/08/22",
|
|
3124
|
+
"instances": [
|
|
3125
|
+
"Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)"
|
|
3126
|
+
],
|
|
3127
|
+
"url": "https://www.seokicks.de/robot.html"
|
|
3128
|
+
},
|
|
3129
|
+
{
|
|
3130
|
+
"pattern": "tracemyfile",
|
|
3131
|
+
"addition_date": "2018/08/23",
|
|
3132
|
+
"instances": [
|
|
3133
|
+
"Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)"
|
|
3134
|
+
]
|
|
3135
|
+
},
|
|
3136
|
+
{
|
|
3137
|
+
"pattern": "Nimbostratus-Bot",
|
|
3138
|
+
"addition_date": "2018/08/29",
|
|
3139
|
+
"instances": [
|
|
3140
|
+
"Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)"
|
|
3141
|
+
]
|
|
3142
|
+
},
|
|
3499
3143
|
{
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3144
|
+
"pattern": "zgrab",
|
|
3145
|
+
"addition_date": "2018/08/30",
|
|
3146
|
+
"instances": [
|
|
3147
|
+
"Mozilla/5.0 zgrab/0.x"
|
|
3148
|
+
],
|
|
3505
3149
|
"url": "https://zmap.io/"
|
|
3506
|
-
}
|
|
3507
|
-
,
|
|
3150
|
+
},
|
|
3508
3151
|
{
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3152
|
+
"pattern": "PR-CY\\.RU",
|
|
3153
|
+
"addition_date": "2018/08/30",
|
|
3154
|
+
"instances": [
|
|
3155
|
+
"Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)"
|
|
3156
|
+
],
|
|
3514
3157
|
"url": "https://a.pr-cy.ru/"
|
|
3515
|
-
}
|
|
3516
|
-
,
|
|
3158
|
+
},
|
|
3517
3159
|
{
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3160
|
+
"pattern": "AdsTxtCrawler",
|
|
3161
|
+
"addition_date": "2018/08/30",
|
|
3162
|
+
"instances": [
|
|
3163
|
+
"AdsTxtCrawler/1.0"
|
|
3164
|
+
]
|
|
3523
3165
|
},
|
|
3524
3166
|
{
|
|
3525
3167
|
"pattern": "Datafeedwatch",
|
|
@@ -3528,8 +3170,7 @@
|
|
|
3528
3170
|
"Datafeedwatch/2.1.x"
|
|
3529
3171
|
],
|
|
3530
3172
|
"url": "https://www.datafeedwatch.com/"
|
|
3531
|
-
}
|
|
3532
|
-
,
|
|
3173
|
+
},
|
|
3533
3174
|
{
|
|
3534
3175
|
"pattern": "Zabbix",
|
|
3535
3176
|
"addition_date": "2018/09/05",
|
|
@@ -3537,8 +3178,7 @@
|
|
|
3537
3178
|
"Zabbix"
|
|
3538
3179
|
],
|
|
3539
3180
|
"url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring"
|
|
3540
|
-
}
|
|
3541
|
-
,
|
|
3181
|
+
},
|
|
3542
3182
|
{
|
|
3543
3183
|
"pattern": "TangibleeBot",
|
|
3544
3184
|
"addition_date": "2018/09/05",
|
|
@@ -3546,8 +3186,7 @@
|
|
|
3546
3186
|
"TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)"
|
|
3547
3187
|
],
|
|
3548
3188
|
"url": "http://tangiblee.com/bot"
|
|
3549
|
-
}
|
|
3550
|
-
,
|
|
3189
|
+
},
|
|
3551
3190
|
{
|
|
3552
3191
|
"pattern": "google-xrawler",
|
|
3553
3192
|
"addition_date": "2018/09/05",
|
|
@@ -3555,8 +3194,7 @@
|
|
|
3555
3194
|
"google-xrawler"
|
|
3556
3195
|
],
|
|
3557
3196
|
"url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for"
|
|
3558
|
-
}
|
|
3559
|
-
,
|
|
3197
|
+
},
|
|
3560
3198
|
{
|
|
3561
3199
|
"pattern": "axios",
|
|
3562
3200
|
"addition_date": "2018/09/06",
|
|
@@ -3565,8 +3203,7 @@
|
|
|
3565
3203
|
"axios/0.19.0"
|
|
3566
3204
|
],
|
|
3567
3205
|
"url": "https://github.com/axios/axios"
|
|
3568
|
-
}
|
|
3569
|
-
,
|
|
3206
|
+
},
|
|
3570
3207
|
{
|
|
3571
3208
|
"pattern": "Amazon CloudFront",
|
|
3572
3209
|
"addition_date": "2018/09/07",
|
|
@@ -3574,16 +3211,14 @@
|
|
|
3574
3211
|
"Amazon CloudFront"
|
|
3575
3212
|
],
|
|
3576
3213
|
"url": "https://aws.amazon.com/cloudfront/"
|
|
3577
|
-
}
|
|
3578
|
-
,
|
|
3214
|
+
},
|
|
3579
3215
|
{
|
|
3580
3216
|
"pattern": "Pulsepoint",
|
|
3581
3217
|
"addition_date": "2018/09/24",
|
|
3582
3218
|
"instances": [
|
|
3583
3219
|
"Pulsepoint XT3 web scraper"
|
|
3584
3220
|
]
|
|
3585
|
-
}
|
|
3586
|
-
,
|
|
3221
|
+
},
|
|
3587
3222
|
{
|
|
3588
3223
|
"pattern": "CloudFlare-AlwaysOnline",
|
|
3589
3224
|
"addition_date": "2018/09/27",
|
|
@@ -3591,27 +3226,24 @@
|
|
|
3591
3226
|
"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34",
|
|
3592
3227
|
"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34"
|
|
3593
3228
|
],
|
|
3594
|
-
"url"
|
|
3595
|
-
}
|
|
3596
|
-
,
|
|
3229
|
+
"url": "https://www.cloudflare.com/always-online/"
|
|
3230
|
+
},
|
|
3597
3231
|
{
|
|
3598
|
-
|
|
3232
|
+
"pattern": "Google-Structured-Data-Testing-Tool",
|
|
3599
3233
|
"addition_date": "2018/10/02",
|
|
3600
3234
|
"instances": [
|
|
3601
3235
|
"Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)",
|
|
3602
3236
|
"Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)"
|
|
3603
3237
|
],
|
|
3604
3238
|
"url": "https://search.google.com/structured-data/testing-tool"
|
|
3605
|
-
}
|
|
3606
|
-
,
|
|
3239
|
+
},
|
|
3607
3240
|
{
|
|
3608
|
-
|
|
3241
|
+
"pattern": "WordupInfoSearch",
|
|
3609
3242
|
"addition_date": "2018/10/07",
|
|
3610
3243
|
"instances": [
|
|
3611
3244
|
"WordupInfoSearch/1.0"
|
|
3612
3245
|
]
|
|
3613
|
-
}
|
|
3614
|
-
,
|
|
3246
|
+
},
|
|
3615
3247
|
{
|
|
3616
3248
|
"pattern": "WebDataStats",
|
|
3617
3249
|
"addition_date": "2018/10/08",
|
|
@@ -3619,16 +3251,14 @@
|
|
|
3619
3251
|
"Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)"
|
|
3620
3252
|
],
|
|
3621
3253
|
"url": "https://webdatastats.com/"
|
|
3622
|
-
}
|
|
3623
|
-
,
|
|
3254
|
+
},
|
|
3624
3255
|
{
|
|
3625
3256
|
"pattern": "HttpUrlConnection",
|
|
3626
3257
|
"addition_date": "2018/10/08",
|
|
3627
3258
|
"instances": [
|
|
3628
3259
|
"Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)"
|
|
3629
3260
|
]
|
|
3630
|
-
}
|
|
3631
|
-
,
|
|
3261
|
+
},
|
|
3632
3262
|
{
|
|
3633
3263
|
"pattern": "ZoomBot",
|
|
3634
3264
|
"addition_date": "2018/10/10",
|
|
@@ -3636,24 +3266,21 @@
|
|
|
3636
3266
|
"ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)"
|
|
3637
3267
|
],
|
|
3638
3268
|
"url": "http://suite.seozoom.it/bot.html"
|
|
3639
|
-
}
|
|
3640
|
-
,
|
|
3269
|
+
},
|
|
3641
3270
|
{
|
|
3642
3271
|
"pattern": "VelenPublicWebCrawler",
|
|
3643
3272
|
"addition_date": "2018/10/09",
|
|
3644
3273
|
"instances": [
|
|
3645
3274
|
"VelenPublicWebCrawler (velen.io)"
|
|
3646
3275
|
]
|
|
3647
|
-
}
|
|
3648
|
-
,
|
|
3276
|
+
},
|
|
3649
3277
|
{
|
|
3650
3278
|
"pattern": "MoodleBot",
|
|
3651
3279
|
"addition_date": "2018/10/10",
|
|
3652
3280
|
"instances": [
|
|
3653
3281
|
"MoodleBot/1.0"
|
|
3654
3282
|
]
|
|
3655
|
-
}
|
|
3656
|
-
,
|
|
3283
|
+
},
|
|
3657
3284
|
{
|
|
3658
3285
|
"pattern": "jpg-newsbot",
|
|
3659
3286
|
"addition_date": "2018/10/10",
|
|
@@ -3661,8 +3288,7 @@
|
|
|
3661
3288
|
"jpg-newsbot/2.0; (+https://vipnytt.no/bots/)"
|
|
3662
3289
|
],
|
|
3663
3290
|
"url": "https://vipnytt.no/bots/"
|
|
3664
|
-
}
|
|
3665
|
-
,
|
|
3291
|
+
},
|
|
3666
3292
|
{
|
|
3667
3293
|
"pattern": "outbrain",
|
|
3668
3294
|
"addition_date": "2018/10/14",
|
|
@@ -3670,8 +3296,7 @@
|
|
|
3670
3296
|
"Mozilla/5.0 (Java) outbrain"
|
|
3671
3297
|
],
|
|
3672
3298
|
"url": "https://www.outbrain.com/help/advertisers/invalid-url/"
|
|
3673
|
-
}
|
|
3674
|
-
,
|
|
3299
|
+
},
|
|
3675
3300
|
{
|
|
3676
3301
|
"pattern": "W3C_Validator",
|
|
3677
3302
|
"addition_date": "2018/10/14",
|
|
@@ -3679,8 +3304,7 @@
|
|
|
3679
3304
|
"W3C_Validator/1.3"
|
|
3680
3305
|
],
|
|
3681
3306
|
"url": "https://validator.w3.org/services"
|
|
3682
|
-
}
|
|
3683
|
-
,
|
|
3307
|
+
},
|
|
3684
3308
|
{
|
|
3685
3309
|
"pattern": "Validator\\.nu",
|
|
3686
3310
|
"addition_date": "2018/10/14",
|
|
@@ -3688,12 +3312,13 @@
|
|
|
3688
3312
|
"Validator.nu/LV"
|
|
3689
3313
|
],
|
|
3690
3314
|
"url": "https://validator.w3.org/services"
|
|
3691
|
-
}
|
|
3692
|
-
,
|
|
3315
|
+
},
|
|
3693
3316
|
{
|
|
3694
3317
|
"pattern": "W3C-checklink",
|
|
3695
3318
|
"addition_date": "2018/10/14",
|
|
3696
|
-
"depends_on": [
|
|
3319
|
+
"depends_on": [
|
|
3320
|
+
"libwww-perl"
|
|
3321
|
+
],
|
|
3697
3322
|
"instances": [
|
|
3698
3323
|
"W3C-checklink/2.90 libwww-perl/5.64",
|
|
3699
3324
|
"W3C-checklink/3.6.2.3 libwww-perl/5.64",
|
|
@@ -3706,8 +3331,7 @@
|
|
|
3706
3331
|
"W3C-checklink/4.5 [4.160] libwww-perl/5.823"
|
|
3707
3332
|
],
|
|
3708
3333
|
"url": "https://validator.w3.org/services"
|
|
3709
|
-
}
|
|
3710
|
-
,
|
|
3334
|
+
},
|
|
3711
3335
|
{
|
|
3712
3336
|
"pattern": "W3C-mobileOK",
|
|
3713
3337
|
"addition_date": "2018/10/14",
|
|
@@ -3715,8 +3339,7 @@
|
|
|
3715
3339
|
"W3C-mobileOK/DDC-1.0"
|
|
3716
3340
|
],
|
|
3717
3341
|
"url": "https://validator.w3.org/services"
|
|
3718
|
-
}
|
|
3719
|
-
,
|
|
3342
|
+
},
|
|
3720
3343
|
{
|
|
3721
3344
|
"pattern": "W3C_I18n-Checker",
|
|
3722
3345
|
"addition_date": "2018/10/14",
|
|
@@ -3724,8 +3347,7 @@
|
|
|
3724
3347
|
"W3C_I18n-Checker/1.0"
|
|
3725
3348
|
],
|
|
3726
3349
|
"url": "https://validator.w3.org/services"
|
|
3727
|
-
}
|
|
3728
|
-
,
|
|
3350
|
+
},
|
|
3729
3351
|
{
|
|
3730
3352
|
"pattern": "FeedValidator",
|
|
3731
3353
|
"addition_date": "2018/10/14",
|
|
@@ -3733,8 +3355,7 @@
|
|
|
3733
3355
|
"FeedValidator/1.3"
|
|
3734
3356
|
],
|
|
3735
3357
|
"url": "https://validator.w3.org/services"
|
|
3736
|
-
}
|
|
3737
|
-
,
|
|
3358
|
+
},
|
|
3738
3359
|
{
|
|
3739
3360
|
"pattern": "W3C_CSS_Validator",
|
|
3740
3361
|
"addition_date": "2018/10/14",
|
|
@@ -3742,8 +3363,7 @@
|
|
|
3742
3363
|
"Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0"
|
|
3743
3364
|
],
|
|
3744
3365
|
"url": "https://validator.w3.org/services"
|
|
3745
|
-
}
|
|
3746
|
-
,
|
|
3366
|
+
},
|
|
3747
3367
|
{
|
|
3748
3368
|
"pattern": "W3C_Unicorn",
|
|
3749
3369
|
"addition_date": "2018/10/14",
|
|
@@ -3751,16 +3371,14 @@
|
|
|
3751
3371
|
"W3C_Unicorn/1.0"
|
|
3752
3372
|
],
|
|
3753
3373
|
"url": "https://validator.w3.org/services"
|
|
3754
|
-
}
|
|
3755
|
-
,
|
|
3374
|
+
},
|
|
3756
3375
|
{
|
|
3757
3376
|
"pattern": "Google-PhysicalWeb",
|
|
3758
3377
|
"addition_date": "2018/10/21",
|
|
3759
3378
|
"instances": [
|
|
3760
3379
|
"Mozilla/5.0 (Google-PhysicalWeb)"
|
|
3761
3380
|
]
|
|
3762
|
-
}
|
|
3763
|
-
,
|
|
3381
|
+
},
|
|
3764
3382
|
{
|
|
3765
3383
|
"pattern": "Blackboard",
|
|
3766
3384
|
"addition_date": "2018/10/28",
|
|
@@ -3840,8 +3458,7 @@
|
|
|
3840
3458
|
"Mozilla/5.0 (compatible; theoldreader.com)"
|
|
3841
3459
|
],
|
|
3842
3460
|
"url": "https://www.theoldreader.com/"
|
|
3843
|
-
}
|
|
3844
|
-
,
|
|
3461
|
+
},
|
|
3845
3462
|
{
|
|
3846
3463
|
"pattern": "AnyEvent",
|
|
3847
3464
|
"addition_date": "2018/12/07",
|
|
@@ -3849,8 +3466,7 @@
|
|
|
3849
3466
|
"Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)"
|
|
3850
3467
|
],
|
|
3851
3468
|
"url": "http://software.schmorp.de/pkg/AnyEvent.html"
|
|
3852
|
-
}
|
|
3853
|
-
,
|
|
3469
|
+
},
|
|
3854
3470
|
{
|
|
3855
3471
|
"pattern": "Uptimebot\\.org",
|
|
3856
3472
|
"addition_date": "2019/01/17",
|
|
@@ -3858,8 +3474,7 @@
|
|
|
3858
3474
|
"Uptimebot.org - Free website monitoring"
|
|
3859
3475
|
],
|
|
3860
3476
|
"url": "http://uptimebot.org/"
|
|
3861
|
-
}
|
|
3862
|
-
,
|
|
3477
|
+
},
|
|
3863
3478
|
{
|
|
3864
3479
|
"pattern": "Nmap Scripting Engine",
|
|
3865
3480
|
"addition_date": "2019/02/04",
|
|
@@ -3867,8 +3482,7 @@
|
|
|
3867
3482
|
"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"
|
|
3868
3483
|
],
|
|
3869
3484
|
"url": "https://nmap.org/book/nse.html"
|
|
3870
|
-
}
|
|
3871
|
-
,
|
|
3485
|
+
},
|
|
3872
3486
|
{
|
|
3873
3487
|
"pattern": "2ip\\.ru",
|
|
3874
3488
|
"addition_date": "2019/02/12",
|
|
@@ -3958,9 +3572,9 @@
|
|
|
3958
3572
|
"url": "https://developers.google.com/web/updates/2017/04/headless-chrome",
|
|
3959
3573
|
"addition_date": "2019/06/17",
|
|
3960
3574
|
"instances": [
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3575
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36",
|
|
3576
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36",
|
|
3577
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36"
|
|
3964
3578
|
]
|
|
3965
3579
|
},
|
|
3966
3580
|
{
|
|
@@ -3978,8 +3592,7 @@
|
|
|
3978
3592
|
"Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)"
|
|
3979
3593
|
],
|
|
3980
3594
|
"url": "http://www.uptime.com/uptimebot"
|
|
3981
|
-
}
|
|
3982
|
-
,
|
|
3595
|
+
},
|
|
3983
3596
|
{
|
|
3984
3597
|
"pattern": "Streamline3Bot\\/",
|
|
3985
3598
|
"addition_date": "2019/07/21",
|
|
@@ -3988,8 +3601,7 @@
|
|
|
3988
3601
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0"
|
|
3989
3602
|
],
|
|
3990
3603
|
"url": "https://www.ubtsupport.com/legal/Streamline3Bot.php"
|
|
3991
|
-
}
|
|
3992
|
-
,
|
|
3604
|
+
},
|
|
3993
3605
|
{
|
|
3994
3606
|
"pattern": "serpstatbot\\/",
|
|
3995
3607
|
"addition_date": "2019/07/25",
|
|
@@ -3998,8 +3610,7 @@
|
|
|
3998
3610
|
"serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)"
|
|
3999
3611
|
],
|
|
4000
3612
|
"url": "http://serpstatbot.com"
|
|
4001
|
-
}
|
|
4002
|
-
,
|
|
3613
|
+
},
|
|
4003
3614
|
{
|
|
4004
3615
|
"pattern": "MixnodeCache\\/",
|
|
4005
3616
|
"addition_date": "2019/08/04",
|
|
@@ -4007,8 +3618,7 @@
|
|
|
4007
3618
|
"MixnodeCache/1.8(+https://cache.mixnode.com/)"
|
|
4008
3619
|
],
|
|
4009
3620
|
"url": "https://cache.mixnode.com/"
|
|
4010
|
-
}
|
|
4011
|
-
,
|
|
3621
|
+
},
|
|
4012
3622
|
{
|
|
4013
3623
|
"pattern": "^curl",
|
|
4014
3624
|
"addition_date": "2019/08/15",
|
|
@@ -4023,8 +3633,7 @@
|
|
|
4023
3633
|
"curl/7.65.3"
|
|
4024
3634
|
],
|
|
4025
3635
|
"url": "https://curl.haxx.se/"
|
|
4026
|
-
}
|
|
4027
|
-
,
|
|
3636
|
+
},
|
|
4028
3637
|
{
|
|
4029
3638
|
"pattern": "SimpleScraper",
|
|
4030
3639
|
"addition_date": "2019/08/16",
|
|
@@ -4032,8 +3641,7 @@
|
|
|
4032
3641
|
"Mozilla/5.0 (compatible; SimpleScraper)"
|
|
4033
3642
|
],
|
|
4034
3643
|
"url": "https://github.com/ramonkcom/simple-scraper/"
|
|
4035
|
-
}
|
|
4036
|
-
,
|
|
3644
|
+
},
|
|
4037
3645
|
{
|
|
4038
3646
|
"pattern": "RSSingBot",
|
|
4039
3647
|
"addition_date": "2019/09/15",
|
|
@@ -4041,8 +3649,7 @@
|
|
|
4041
3649
|
"RSSingBot (http://www.rssing.com)"
|
|
4042
3650
|
],
|
|
4043
3651
|
"url": "http://www.rssing.com"
|
|
4044
|
-
}
|
|
4045
|
-
,
|
|
3652
|
+
},
|
|
4046
3653
|
{
|
|
4047
3654
|
"pattern": "Jooblebot",
|
|
4048
3655
|
"addition_date": "2019/09/25",
|
|
@@ -4050,8 +3657,7 @@
|
|
|
4050
3657
|
"Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36"
|
|
4051
3658
|
],
|
|
4052
3659
|
"url": "http://jooble.org/jooble-bot"
|
|
4053
|
-
}
|
|
4054
|
-
,
|
|
3660
|
+
},
|
|
4055
3661
|
{
|
|
4056
3662
|
"pattern": "fedoraplanet",
|
|
4057
3663
|
"addition_date": "2019/09/28",
|
|
@@ -4059,8 +3665,7 @@
|
|
|
4059
3665
|
"venus/fedoraplanet"
|
|
4060
3666
|
],
|
|
4061
3667
|
"url": "http://fedoraplanet.org/"
|
|
4062
|
-
}
|
|
4063
|
-
,
|
|
3668
|
+
},
|
|
4064
3669
|
{
|
|
4065
3670
|
"pattern": "Friendica",
|
|
4066
3671
|
"addition_date": "2019/09/28",
|
|
@@ -4068,8 +3673,7 @@
|
|
|
4068
3673
|
"Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz"
|
|
4069
3674
|
],
|
|
4070
3675
|
"url": "https://hoyer.xyz"
|
|
4071
|
-
}
|
|
4072
|
-
,
|
|
3676
|
+
},
|
|
4073
3677
|
{
|
|
4074
3678
|
"pattern": "NextCloud",
|
|
4075
3679
|
"addition_date": "2019/09/30",
|
|
@@ -4077,8 +3681,7 @@
|
|
|
4077
3681
|
"NextCloud-News/1.0"
|
|
4078
3682
|
],
|
|
4079
3683
|
"url": "https://nextcloud.com/"
|
|
4080
|
-
}
|
|
4081
|
-
,
|
|
3684
|
+
},
|
|
4082
3685
|
{
|
|
4083
3686
|
"pattern": "Tiny Tiny RSS",
|
|
4084
3687
|
"addition_date": "2019/10/04",
|
|
@@ -4089,8 +3692,7 @@
|
|
|
4089
3692
|
"Tiny Tiny RSS/19.8 (http://tt-rss.org/)"
|
|
4090
3693
|
],
|
|
4091
3694
|
"url": "http://tt-rss.org/"
|
|
4092
|
-
}
|
|
4093
|
-
,
|
|
3695
|
+
},
|
|
4094
3696
|
{
|
|
4095
3697
|
"pattern": "RegionStuttgartBot",
|
|
4096
3698
|
"addition_date": "2019/10/17",
|
|
@@ -4098,35 +3700,33 @@
|
|
|
4098
3700
|
"Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)"
|
|
4099
3701
|
],
|
|
4100
3702
|
"url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/"
|
|
4101
|
-
}
|
|
4102
|
-
,
|
|
3703
|
+
},
|
|
4103
3704
|
{
|
|
4104
3705
|
"pattern": "Bytespider",
|
|
4105
3706
|
"addition_date": "2019/11/11",
|
|
4106
3707
|
"instances": [
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
3708
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider",
|
|
3709
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider",
|
|
3710
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com",
|
|
3711
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider",
|
|
3712
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider",
|
|
3713
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider",
|
|
3714
|
+
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider",
|
|
3715
|
+
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider",
|
|
3716
|
+
"Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider",
|
|
3717
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider",
|
|
3718
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider",
|
|
3719
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider",
|
|
3720
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider",
|
|
3721
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider",
|
|
3722
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider",
|
|
3723
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider",
|
|
3724
|
+
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider",
|
|
3725
|
+
"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider",
|
|
3726
|
+
"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider"
|
|
4126
3727
|
],
|
|
4127
3728
|
"url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent"
|
|
4128
|
-
}
|
|
4129
|
-
,
|
|
3729
|
+
},
|
|
4130
3730
|
{
|
|
4131
3731
|
"pattern": "Datanyze",
|
|
4132
3732
|
"addition_date": "2019/11/17",
|
|
@@ -4134,8 +3734,7 @@
|
|
|
4134
3734
|
"Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
|
|
4135
3735
|
],
|
|
4136
3736
|
"url": "https://www.datanyze.com/dnyzbot/"
|
|
4137
|
-
}
|
|
4138
|
-
,
|
|
3737
|
+
},
|
|
4139
3738
|
{
|
|
4140
3739
|
"pattern": "Google-Site-Verification",
|
|
4141
3740
|
"addition_date": "2019/12/11",
|
|
@@ -4143,8 +3742,7 @@
|
|
|
4143
3742
|
"Mozilla/5.0 (compatible; Google-Site-Verification/1.0)"
|
|
4144
3743
|
],
|
|
4145
3744
|
"url": "https://support.google.com/webmasters/answer/9008080"
|
|
4146
|
-
}
|
|
4147
|
-
,
|
|
3745
|
+
},
|
|
4148
3746
|
{
|
|
4149
3747
|
"pattern": "TrendsmapResolver",
|
|
4150
3748
|
"addition_date": "2020/02/24",
|
|
@@ -4152,8 +3750,7 @@
|
|
|
4152
3750
|
"Mozilla/5.0 (compatible; TrendsmapResolver/0.1)"
|
|
4153
3751
|
],
|
|
4154
3752
|
"url": "https://www.trendsmap.com/"
|
|
4155
|
-
}
|
|
4156
|
-
,
|
|
3753
|
+
},
|
|
4157
3754
|
{
|
|
4158
3755
|
"pattern": "tweetedtimes",
|
|
4159
3756
|
"addition_date": "2020/02/24",
|
|
@@ -4647,8 +4244,8 @@
|
|
|
4647
4244
|
"pattern": "virustotal",
|
|
4648
4245
|
"addition_date": "2021/09/22",
|
|
4649
4246
|
"instances": [
|
|
4650
|
-
"Mozilla
|
|
4651
|
-
"AppEngine-Google; (+http
|
|
4247
|
+
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)",
|
|
4248
|
+
"AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)"
|
|
4652
4249
|
],
|
|
4653
4250
|
"url": "https://www.virustotal.com/gui/home/url"
|
|
4654
4251
|
},
|
|
@@ -4677,14 +4274,14 @@
|
|
|
4677
4274
|
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36"
|
|
4678
4275
|
]
|
|
4679
4276
|
},
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4277
|
+
{
|
|
4278
|
+
"pattern": "trovitBot",
|
|
4279
|
+
"addition_date": "2022/06/08",
|
|
4280
|
+
"url": "http://www.trovit.com/bot.html",
|
|
4281
|
+
"instances": [
|
|
4282
|
+
"Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)"
|
|
4283
|
+
]
|
|
4284
|
+
},
|
|
4688
4285
|
{
|
|
4689
4286
|
"pattern": "seostar\\.co",
|
|
4690
4287
|
"addition_date": "2022/08/04",
|
|
@@ -4762,7 +4359,9 @@
|
|
|
4762
4359
|
"pattern": "RuxitSynthetic",
|
|
4763
4360
|
"addition_date": "2023/02/16",
|
|
4764
4361
|
"url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent",
|
|
4765
|
-
"instances"
|
|
4362
|
+
"instances": [
|
|
4363
|
+
"RuxitSynthetic/1.0"
|
|
4364
|
+
]
|
|
4766
4365
|
},
|
|
4767
4366
|
{
|
|
4768
4367
|
"pattern": "Google-Read-Aloud",
|
|
@@ -4785,7 +4384,7 @@
|
|
|
4785
4384
|
"addition_date": "2023/05/24",
|
|
4786
4385
|
"instances": [
|
|
4787
4386
|
"OdklBot/1.0 (share@odnoklassniki.ru)",
|
|
4788
|
-
|
|
4387
|
+
"Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)"
|
|
4789
4388
|
],
|
|
4790
4389
|
"url": "https://odnoklassniki.ru/"
|
|
4791
4390
|
},
|
|
@@ -4876,7 +4475,7 @@
|
|
|
4876
4475
|
"addition_date": "2023/08/24",
|
|
4877
4476
|
"instances": [
|
|
4878
4477
|
"Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)",
|
|
4879
|
-
|
|
4478
|
+
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36"
|
|
4880
4479
|
],
|
|
4881
4480
|
"url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en"
|
|
4882
4481
|
},
|
|
@@ -5304,19 +4903,25 @@
|
|
|
5304
4903
|
{
|
|
5305
4904
|
"pattern": "Monsidobot",
|
|
5306
4905
|
"addition_date": "2024/05/14",
|
|
5307
|
-
"instances": [
|
|
4906
|
+
"instances": [
|
|
4907
|
+
"Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)"
|
|
4908
|
+
],
|
|
5308
4909
|
"url": "http://monsido.com/bot.html"
|
|
5309
4910
|
},
|
|
5310
4911
|
{
|
|
5311
4912
|
"pattern": "GroupMeBot",
|
|
5312
4913
|
"addition_date": "2024/05/19",
|
|
5313
|
-
"instances": [
|
|
4914
|
+
"instances": [
|
|
4915
|
+
"GroupMeBot/1.0"
|
|
4916
|
+
],
|
|
5314
4917
|
"url": "https://groupme.com/"
|
|
5315
4918
|
},
|
|
5316
4919
|
{
|
|
5317
4920
|
"pattern": "Vercelbot",
|
|
5318
4921
|
"addition_date": "2024/08/30",
|
|
5319
|
-
"instances": [
|
|
4922
|
+
"instances": [
|
|
4923
|
+
"Vercelbot (+https://vercel.com)"
|
|
4924
|
+
],
|
|
5320
4925
|
"url": "https://github.com/vercel/vercel/discussions/5095#discussioncomment-58705"
|
|
5321
4926
|
},
|
|
5322
4927
|
{
|