crawler_detect 0.1.5 → 0.1.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9f469a4f042b4ce7600daf8732b42a0943af6c466f028db67366381846155df1
4
- data.tar.gz: 469b493cacd0d76d0a9bfb94d7905dc39298d297a6eff084c3461351c22e60ce
3
+ metadata.gz: d59268ab3ed03de60b67ac58bc9f1891b131f282cefdd32344a7ae633061700b
4
+ data.tar.gz: c2efe88dc14eb589636ae11f0ef0e78d36091c8de449ea8884a6ba81f545459a
5
5
  SHA512:
6
- metadata.gz: 5ac0fe561eea7de310abc90f2265770218f004d7c069baaf80e22ecee02ef1b9ed01dc68b2b35e166738320e3d7c269f156bae4580e74c05ed505f64d6942596
7
- data.tar.gz: dafeb5060813d91c75a1ffbcf063f5fd229b635ea3d83883e5cf546fe6286fa68e5ea1c9834caeda40997e43c0494e6fc90f842d680eaaeca783517c967e2cb2
6
+ metadata.gz: fb23c32b42868dfe6c687c6b20d44312b35031a54ec0570f43bb209db5bcf8b94193500057aea3b69df61e6a7b5c947e2bce552308be2b23530e7562a8316f56
7
+ data.tar.gz: c74a44a447eab8c6665ff556a21d1eef3757fb5e18bb585dca852a1c3081b9850a81c5927bbf5e5c11324ce55cdbd09b20a9f3657fad5bac12d3e35c399fab31
@@ -25,7 +25,7 @@ module CrawlerDetect
25
25
  end
26
26
 
27
27
  def matches_crawler_list?
28
- @crawler_name = crawlers_matcher.match(@user_agent).to_s
28
+ @crawler_name = crawlers_matcher.match(@user_agent).to_s.strip
29
29
  !@crawler_name.empty?
30
30
  end
31
31
 
@@ -37,7 +37,6 @@ module CrawlerDetect
37
37
  13TABS
38
38
  192\.comAgent
39
39
  2ip\.ru
40
- 404checker
41
40
  404enemy
42
41
  7Siters
43
42
  80legs
@@ -55,42 +54,41 @@ AdAuth\/
55
54
  adbeat
56
55
  AddThis
57
56
  ADmantX
57
+ AdminLabs
58
58
  adressendeutschland
59
- adscanner\/
60
- Advanced Email Extractor v
59
+ adscanner
61
60
  agentslug
62
61
  AHC
63
62
  aihit
64
63
  aiohttp\/
65
64
  Airmail
66
- Akamai_Site_Analyzer
67
65
  akka-http\/
68
66
  akula\/
69
67
  alertra
70
68
  alexa site audit
71
69
  Alibaba\.Security\.Heimdall
72
70
  Alligator
73
- allloadin\.com
71
+ allloadin
74
72
  AllSubmitter
75
73
  alyze\.info
76
74
  amagit
77
75
  Anarchie
78
76
  AndroidDownloadManager
79
77
  Anemone
80
- AngleSharp\/
78
+ AngleSharp
81
79
  annotate_google
82
80
  Ant\.com
83
81
  Anturis Agent
84
82
  AnyEvent-HTTP\/
85
83
  Apache Droid
86
84
  Apache OpenOffice
87
- Apache-HttpAsyncClient\/
88
- Apache-HttpClient\/
89
- ApacheBench\/
85
+ Apache-HttpAsyncClient
86
+ Apache-HttpClient
87
+ ApacheBench
90
88
  Apexoo
91
89
  APIs-Google
92
- AportWorm\/[0-9]
93
- AppBeat\/[0-9]
90
+ AportWorm\/
91
+ AppBeat\/
94
92
  AppEngine-Google
95
93
  AppStoreScraperZ
96
94
  Aprc\/[0-9]
@@ -99,7 +97,7 @@ arachnode
99
97
  Arachnophilia
100
98
  aria2
101
99
  Arukereso
102
- asafaweb\.com
100
+ asafaweb
103
101
  AskQuickly
104
102
  Ask Jeeves
105
103
  ASPSeek
@@ -122,15 +120,14 @@ baidu\.com
122
120
  Bandit
123
121
  basicstate
124
122
  BatchFTP
125
- Battleztar\ Bazinga
126
- baypup\/[0-9]
127
- baypup\/colbert
123
+ Battlezta Bazinga
124
+ baypup\/
128
125
  BazQux
129
126
  BBBike
130
127
  BCKLINKS
131
128
  BDFetch
132
- BegunAdvertising\/
133
- Bidtellect\/
129
+ BegunAdvertising
130
+ Bidtellect
134
131
  BigBozz
135
132
  Bigfoot
136
133
  biglotron
@@ -140,24 +137,23 @@ binlar
140
137
  biNu image cacher
141
138
  Bitacle
142
139
  biz_Directory
143
- Black\ Hole
140
+ Black Hole
144
141
  Blackboard Safeassign
145
142
  BlackWidow
146
143
  BlockNote\.Net
147
- Bloglines\/
144
+ Bloglines
148
145
  Bloglovin
149
146
  BlogPulseLive
150
147
  BlogSearch
151
148
  Blogtrottr
152
149
  BlowFish
153
- Boardreader
154
150
  boitho\.com-dc
155
151
  BPImageWalker
156
152
  Braintree-Webhooks
157
153
  Branch Metrics API
158
154
  Branch-Passthrough
159
155
  Brandprotect
160
- BrandVerity\/[0-9]
156
+ BrandVerity
161
157
  Brandwatch
162
158
  Brodie\/
163
159
  Browsershots
@@ -174,7 +170,6 @@ CAAM\/[0-9]
174
170
  CakePHP
175
171
  Calculon
176
172
  Canary%20Mail
177
- CapsuleChecker
178
173
  CaretNail
179
174
  catexplorador
180
175
  CC Metadata Scaper
@@ -184,37 +179,37 @@ Cerberian Drtrs
184
179
  CERT\.at-Statistics-Survey
185
180
  cg-eye
186
181
  changedetection
187
- ChangesMeter\/
182
+ ChangesMeter
188
183
  Charlotte
189
184
  CheckHost
190
185
  checkprivacy
191
186
  CherryPicker
192
187
  ChinaClaw
193
- Chirp\/[0-9]
188
+ Chirp\/
194
189
  chkme\.com
195
190
  Chlooe
196
191
  Chromaxa
197
- CirrusExplorer\/
192
+ CirrusExplorer
198
193
  CISPA Vulnerability Notification
199
194
  Citoid
200
195
  CJNetworkQuality
201
196
  Clarsentia
202
197
  clips\.ua\.ac\.be
203
- Cloud\ mapping
198
+ Cloud mapping
204
199
  CloudEndure
205
200
  CloudFlare-AlwaysOnline
206
- Cloudinary\/[0-9]
201
+ Cloudinary
207
202
  cmcm\.com
208
203
  coccoc
209
204
  cognitiveseo
210
205
  colly -
211
206
  CommaFeed
212
207
  Commons-HttpClient
213
- Comodo SSL Checker
208
+ commonscan
214
209
  contactbigdatafr
215
210
  contentkingapp
216
211
  convera
217
- CookieReports\.com
212
+ CookieReports
218
213
  copyright sheriff
219
214
  CopyRightCheck
220
215
  Copyscape
@@ -227,7 +222,7 @@ CSHttp
227
222
  curb
228
223
  Curious George
229
224
  curl
230
- cuwhois\/[0-9]
225
+ cuwhois\/
231
226
  cybo\.com
232
227
  DAP\/NetHTTP
233
228
  DareBoost
@@ -239,7 +234,6 @@ DataparkSearch
239
234
  dataprovider
240
235
  DataXu
241
236
  Daum(oa)?[ \/][0-9]
242
- DemandbasePublisherAnalyzer\/
243
237
  Demon
244
238
  DeuSu
245
239
  developers\.google\.com\/\+\/web\/snippet\/
@@ -253,23 +247,21 @@ Dispatch\/
253
247
  DittoSpyder
254
248
  dlvr
255
249
  DMBrowser
256
- DNS-Tools Header-Analyzer
257
250
  DNSPod-reporting
258
251
  docoloc
259
- Dolphin http client\/
252
+ Dolphin http client
260
253
  DomainAppender
261
254
  Donuts Content Explorer
262
255
  dotMailer content retrieval
263
256
  dotSemantic
264
257
  downforeveryoneorjustme
265
- Download\ Wonder
266
- downnotifier\.com
258
+ Download Wonder
259
+ downnotifier
267
260
  DowntimeDetector
268
- Dragonfly File Reader
269
261
  Drip
270
262
  drupact
271
263
  Drupal \(\+http:\/\/drupal\.org\/\)
272
- DTS\ Agent
264
+ DTS Agent
273
265
  dubaiindex
274
266
  EARTHCOM
275
267
  Easy-Thumb
@@ -285,8 +277,7 @@ EirGrabber
285
277
  ElectricMonk
286
278
  elefent
287
279
  EMail Exractor
288
- EMail\ Wolf
289
- Email%20Extractor
280
+ EMail Wolf
290
281
  EmailWolf
291
282
  Embarcadero
292
283
  Embed PHP Library
@@ -304,8 +295,7 @@ Excel\/
304
295
  exif
305
296
  Exploratodo
306
297
  Express WebPictures
307
- ExtractorPro
308
- Extreme\ Picture\ Finder
298
+ Extreme Picture Finder
309
299
  EyeNetIE
310
300
  ezooms
311
301
  facebookexternalhit
@@ -322,14 +312,13 @@ Feedable\/
322
312
  Feedbin
323
313
  FeedBooster
324
314
  FeedBucket
325
- FeedBunch\/[0-9]
315
+ FeedBunch\/
326
316
  FeedBurner
327
- FeedChecker
317
+ feeder
328
318
  Feedly
329
- Feedreader
330
319
  FeedshowOnline
331
320
  Feedspot
332
- Feedwind\/[0-9]
321
+ Feedwind\/
333
322
  FeedZcollector
334
323
  feeltiptop
335
324
  Fetch API
@@ -348,33 +337,30 @@ fluffy
348
337
  Flunky
349
338
  flynxapp
350
339
  forensiq
351
- FoundSeoTool\/[0-9]
340
+ FoundSeoTool
352
341
  http:\/\/www.neomo.de\/
353
342
  free thumbnails
354
343
  Freeuploader
355
- FreeWebMonitoring SiteChecker
356
344
  Funnelback
357
345
  G-i-g-a-b-o-t
358
346
  g00g1e\.net
359
- GAChecker
360
- ganarvisitas\/[0-9]
347
+ ganarvisitas
361
348
  geek-tools
362
- Genderanalyzer
363
349
  Genieo
364
350
  GentleSource
365
351
  GetCode
366
352
  Getintent
367
353
  GetLinkInfo
368
- getprismatic\.com
354
+ getprismatic
369
355
  GetRight
370
356
  getroot
371
- GetURLInfo\/[0-9]
357
+ GetURLInfo\/
372
358
  GetWeb
373
359
  Ghost Inspector
374
360
  GigablastOpenSource
375
361
  GIS-LABS
376
362
  github-camo
377
- github\.com\/
363
+ github\.com
378
364
  Go [\d\.]* package http
379
365
  Go http package
380
366
  Go-Ahead-Got-It
@@ -384,7 +370,7 @@ gobyus
384
370
  gofetch
385
371
  GomezAgent
386
372
  gooblog
387
- Goodzer\/[0-9]
373
+ Goodzer\/
388
374
  Google AppsViewer
389
375
  Google Desktop
390
376
  Google favicon
@@ -394,7 +380,6 @@ Google Page Speed Insights
394
380
  Google PP Default
395
381
  Google Search Console
396
382
  Google Web Preview
397
- google_partner_monitoring
398
383
  Google-Adwords
399
384
  Google-Apps-Script
400
385
  Google-Calendar-Importer
@@ -406,7 +391,6 @@ Google-Site-Verification
406
391
  Google-Structured-Data-Testing-Tool
407
392
  Google-Youtube-Links
408
393
  google-xrawler
409
- GoogleCloudMonitoring
410
394
  GoogleDocs
411
395
  GoogleHC\/
412
396
  GoogleProducer
@@ -415,7 +399,6 @@ Google-Transparency-Report
415
399
  Gookey
416
400
  GoScraper
417
401
  GoSpotCheck
418
- GoSquared-Status-Checker
419
402
  gosquared-thumbnailer
420
403
  Gotit
421
404
  GoZilla
@@ -438,29 +421,29 @@ HAA(A)?RTLAND http client
438
421
  Haansoft
439
422
  hackney\/
440
423
  Hadi Agent
424
+ HappyApps-WebCheck
441
425
  Hatena
442
426
  Havij
443
- hawkReader
444
427
  HeadlessChrome
445
428
  HEADMasterSEO
446
429
  HeartRails_Capture
447
430
  help@dataminr\.com
448
431
  heritrix
449
- historious\/
432
+ historious
450
433
  hkedcity
451
- hledejLevne\.cz\/[0-9]
434
+ hledejLevne\.cz
452
435
  Hloader
453
436
  HMView
454
437
  Holmes
455
- HonesoSearchEngine\/
438
+ HonesoSearchEngine
456
439
  HootSuite Image proxy
457
- Hootsuite-WebFeed\/[0-9]
440
+ Hootsuite-WebFeed
458
441
  hosterstats
459
442
  HostTracker
460
443
  ht:\/\/check
461
444
  htdig
462
445
  HTMLparser
463
- htmlyse\.com
446
+ htmlyse
464
447
  HTTP Banner Detection
465
448
  HTTP_Compression_Test
466
449
  http_request2
@@ -485,19 +468,18 @@ httrack
485
468
  huaweisymantec
486
469
  HubSpot
487
470
  Humanlinks
488
- HyperZbozi\.cz Feeder
489
471
  i2kconnect\/
490
472
  Iblog
491
473
  ichiro
492
474
  Id-search
493
475
  IdeelaborPlagiaat
494
476
  IDG Twitter Links Resolver
495
- IDwhois\/[0-9]
477
+ IDwhois\/
496
478
  Iframely
497
479
  igdeSpyder
498
480
  IlTrovatore
499
- Image\ Fetch
500
- Image\ Sucker
481
+ Image Fetch
482
+ Image Sucker
501
483
  ImageEngine\/
502
484
  ImageVisu\/
503
485
  Imagga
@@ -506,12 +488,12 @@ imgsizer
506
488
  InAGist
507
489
  inbound\.li parser
508
490
  InDesign%20CC
509
- Indy\ Library
491
+ Indy Library
510
492
  InetURL
511
493
  infegy
512
494
  infohelfer
513
495
  InfoTekies
514
- InfoWizards Reciprocal Link System PRO
496
+ InfoWizards Reciprocal Link
515
497
  inpwrd\.com
516
498
  instabid
517
499
  Instapaper
@@ -520,7 +502,7 @@ integromedb
520
502
  Intelliseek
521
503
  InterGET
522
504
  internet_archive
523
- Internet\ Ninja
505
+ Internet Ninja
524
506
  InternetSeer
525
507
  internetVista monitor
526
508
  intraVnews
@@ -536,7 +518,7 @@ Irokez
536
518
  isitup\.org
537
519
  iskanie
538
520
  isUp\.li
539
- iThemes Sync\/[0-9]
521
+ iThemes Sync\/
540
522
  iZSearch
541
523
  JAHHO
542
524
  janforman
@@ -558,12 +540,10 @@ Jorgee
558
540
  JS-Kit
559
541
  JustView
560
542
  Kaspersky Lab CFR link resolver
561
- KeepRight OpenStreetMap Checker
562
543
  Kelny\/
563
544
  Kerrigan\/
564
545
  KeyCDN
565
- Keyword Extractor
566
- Keyword\ Density
546
+ Keyword Density
567
547
  Keywords Research
568
548
  KickFire
569
549
  KimonoLabs\/
@@ -577,7 +557,6 @@ KumKie
577
557
  L\.webis
578
558
  Larbin
579
559
  Lavf\/
580
- LayeredExtractor
581
560
  LeechFTP
582
561
  LeechGet
583
562
  letsencrypt
@@ -590,7 +569,6 @@ Licorne
590
569
  Liferea\/
591
570
  Lightspeedsystems
592
571
  Likse
593
- link checker
594
572
  Link Valet
595
573
  link_thumbnailer
596
574
  LinkAlarm\/
@@ -630,15 +608,14 @@ MapperCmd
630
608
  marketinggrader
631
609
  MarkMonitor
632
610
  MarkWatch
633
- Mass\ Downloader
634
- masscan\/[0-9]
635
- Mata\ Hari
611
+ Mass Downloader
612
+ masscan\/
613
+ Mata Hari
636
614
  Mediapartners-Google
637
615
  mediawords
638
616
  MegaIndex\.ru
639
617
  MeltwaterNews
640
- Melvil Rawi\/
641
- MergeFlow-PageReader
618
+ Melvil Rawi
642
619
  Metaspinner
643
620
  MetaURI
644
621
  MFC_Tear_Sample
@@ -647,13 +624,13 @@ Microsoft Office
647
624
  Microsoft Outlook
648
625
  Microsoft Windows Network Diagnostics
649
626
  Microsoft-WebDAV-MiniRedir
650
- Microsoft\ Data\ Access
651
- MIDown\ tool
627
+ Microsoft Data Access
628
+ MIDown tool
652
629
  MIIxpc
653
630
  Mindjet
654
- Miniature\.io\/
631
+ Miniature\.io
655
632
  Miniflux
656
- Mister\ PiX
633
+ Mister PiX
657
634
  mixdata dot com
658
635
  mixed-content-scan
659
636
  Mixmax-LinkPreview
@@ -664,16 +641,16 @@ Mojeek
664
641
  Mojolicious \(Perl\)
665
642
  Monit\/
666
643
  monitis
667
- Monitority\/[0-9]
644
+ Monitority\/
668
645
  montastic
669
646
  MonTools
670
647
  Moreover
671
- Morfeus\ Fucking\ Scanner
648
+ Morfeus Fucking Scanner
672
649
  Morning Paper
673
650
  MovableType
674
651
  mowser
675
652
  Mrcgiguy
676
- MS\ Web\ Services\ Client\ Protocol
653
+ MS Web Services Client Protocol
677
654
  MSFrontPage
678
655
  mShots
679
656
  MuckRack\/
@@ -681,14 +658,14 @@ muhstik-scan
681
658
  MVAClient
682
659
  MxToolbox\/
683
660
  nagios
684
- Najdi\.si\/
685
- Name\ Intelligence
661
+ Najdi\.si
662
+ Name Intelligence
686
663
  Nameprotect
687
664
  Navroad
688
665
  NearSite
689
666
  Needle
690
667
  Nessus
691
- Net\ Vampire
668
+ Net Vampire
692
669
  NetAnts
693
670
  NETCRAFT
694
671
  NetLyzer
@@ -713,11 +690,11 @@ NG-Search
713
690
  Nibbler
714
691
  NICErsPRO
715
692
  Nikto
716
- nineconnections\.com
693
+ nineconnections
717
694
  NLNZ_IAHarvester
718
695
  Nmap Scripting Engine
719
696
  node-superagent
720
- node-urllib\/
697
+ node-urllib
721
698
  node\.io
722
699
  Nodemeter
723
700
  NodePing
@@ -729,16 +706,16 @@ nuhk
729
706
  nutch
730
707
  Nuzzel
731
708
  nWormFeedFinder
709
+ nyawc\/
732
710
  Nymesis
733
711
  NYU
734
- Ocelli\/[0-9]
712
+ Ocelli\/
735
713
  Octopus
736
714
  oegp
737
715
  Offline Explorer
738
- Offline\ Navigator
739
- og-scraper\/
716
+ Offline Navigator
717
+ og-scraper
740
718
  okhttp
741
- Omea Reader
742
719
  omgili
743
720
  OMSC
744
721
  Online Domain Tools
@@ -749,26 +726,25 @@ Openstat\/
749
726
  OpenVAS
750
727
  Optimizer
751
728
  Orbiter
752
- OrgProbe\/[0-9]
729
+ OrgProbe\/
753
730
  orion-semantics
754
731
  Outlook-Express
732
+ Outlook-iOS
755
733
  ow\.ly
756
734
  Owler
757
735
  ownCloud News
758
- OxfordCloudService\/[0-9]
759
- Page Analyzer
736
+ OxfordCloudService
760
737
  Page Valet
761
738
  page_verifier
762
- page\ scorer
739
+ page scorer
763
740
  page2rss
764
- PageAnalyzer
765
741
  PageGrabber
766
742
  PagePeeker
767
743
  PageScorer
768
- Pagespeed\/[0-9]
744
+ Pagespeed\/
769
745
  Panopta
770
746
  panscient
771
- Papa\ Foto
747
+ Papa Foto
772
748
  parsijoo
773
749
  Pavuk
774
750
  PayPal IPN
@@ -789,7 +765,7 @@ Picscout
789
765
  Picsearch
790
766
  PictureFinder
791
767
  Pimonster
792
- ping\.blo\.gs\/
768
+ ping\.blo\.gs
793
769
  Pingability
794
770
  PingAdmin\.Ru
795
771
  Pingdom
@@ -806,21 +782,21 @@ PocketImageCache
806
782
  PocketParser
807
783
  Pockey
808
784
  POE-Component-Client-HTTP
785
+ Polymail\/
809
786
  Pompos
810
787
  Porkbun
811
788
  Port Monitor
812
789
  postano
813
- PostmanRuntime\/
790
+ PostmanRuntime
814
791
  PostPost
815
792
  postrank
816
793
  PowerPoint\/
817
794
  Priceonomics Analysis Engine
818
- PrintFriendly\.com
819
- PritTorrent\/[0-9]
795
+ PrintFriendly
796
+ PritTorrent
820
797
  Prlog
821
798
  probethenet
822
799
  Project 25499
823
- Promotion_Tools_www\.searchenginepromotionhelp\.com
824
800
  prospectb2b
825
801
  Protopage
826
802
  ProWebWalker
@@ -838,8 +814,8 @@ Qirina Hurdler
838
814
  QQDownload
839
815
  QrafterPro
840
816
  Qseero
841
- Qualidator\.com SiteAnalyzer
842
- QueryN\ Metasearch
817
+ Qualidator
818
+ QueryN Metasearch
843
819
  queuedriver
844
820
  Quora Link Preview
845
821
  Qwantify
@@ -855,22 +831,19 @@ RebelMouse
855
831
  Recorder
856
832
  RecurPost\/
857
833
  redback\/
858
- Redirect Checker Tool
859
834
  ReederForMac
860
835
  ReGet
861
836
  RepoMonkey
862
837
  request\.js
863
- ResponseCodeTest\/[0-9]
838
+ ResponseCodeTest
864
839
  RestSharp
865
840
  Riddler
866
841
  Rival IQ
867
842
  Robosourcer
868
- Robozilla\/[0-9]
843
+ Robozilla
869
844
  ROI Hunter
870
845
  RPT-HTTPClient
871
- rss reader
872
846
  RSSOwl
873
- RssReader\/
874
847
  safe-agent-scanner
875
848
  SalesIntelligent
876
849
  Saleslift
@@ -888,11 +861,12 @@ ScrapeBox Page Scanner
888
861
  SimpleScraper
889
862
  Scrapy
890
863
  Screaming
891
- ScreenShotService\/[0-9]
864
+ ScreenShotService
892
865
  Scrubby
893
866
  Scrutiny\/
894
867
  search\.thunderstone
895
- Search37\/
868
+ Search37
869
+ searchenginepromotionhelp
896
870
  Searchestate
897
871
  SearchExpress
898
872
  SearchSight
@@ -912,29 +886,26 @@ SeoCheck
912
886
  SEOkicks
913
887
  Seomoz
914
888
  SEOprofiler
915
- SEOsearch\/
916
- SeopultContentAnalyzer
889
+ SEOsearch
917
890
  seoscanners
891
+ seositecheckup
918
892
  SEOstats
919
- Server Density Service Monitoring
920
- servernfo\.com
893
+ servernfo
921
894
  sexsearcher
922
895
  Seznam
923
896
  Shelob
924
897
  Shodan
925
- Shoppimon Analyzer
926
- ShoppimonAgent\/[0-9]
898
+ Shoppimon
927
899
  ShopWiki
928
900
  ShortLinkTranslate
929
901
  shrinktheweb
930
902
  Sideqik
931
- SilverReader
932
903
  SimplePie
933
904
  SimplyFast
934
905
  Siphon
935
906
  SISTRIX
936
907
  Site-Shot\/
937
- Site\ Sucker
908
+ Site Sucker
938
909
  Site24x7
939
910
  SiteBar
940
911
  Sitebeam
@@ -1002,7 +973,6 @@ StumbleUpon
1002
973
  suchen
1003
974
  Sucuri
1004
975
  summify
1005
- Super Monitoring
1006
976
  SuperHTTP
1007
977
  Surphace Scout
1008
978
  Suzuran
@@ -1024,16 +994,16 @@ Telesphoreo
1024
994
  Telesphorep
1025
995
  Tenon\.io
1026
996
  teoma
1027
- terrainformatica\.com
997
+ terrainformatica
1028
998
  Test Certificate Info
1029
- Tetrahedron\/[0-9]
999
+ testuri
1000
+ Tetrahedron
1030
1001
  The Drop Reaper
1031
1002
  The Expert HTML Source Viewer
1032
1003
  The Knowledge AI
1033
- The\ Intraformant
1004
+ The Intraformant
1034
1005
  theinternetrules
1035
1006
  TheNomad
1036
- theoldreader\.com
1037
1007
  Thinklab
1038
1008
  Thumbshots
1039
1009
  ThumbSniper
@@ -1049,13 +1019,13 @@ tracemyfile
1049
1019
  TrapitAgent
1050
1020
  Trendiction
1051
1021
  Trendsmap
1052
- trendspottr\.com
1022
+ trendspottr
1053
1023
  truwoGPS
1054
1024
  TryJsoup
1055
1025
  TulipChain
1056
1026
  Turingos
1057
1027
  Turnitin
1058
- tweetedtimes\.com
1028
+ tweetedtimes
1059
1029
  Tweetminster
1060
1030
  Tweezler\/
1061
1031
  twibble
@@ -1077,16 +1047,15 @@ updown\.io daemon
1077
1047
  Upflow
1078
1048
  Uptimia
1079
1049
  URL Verifier
1080
- URLChecker
1081
- URLitor\.com
1050
+ URLitor
1082
1051
  urlresolver
1083
1052
  Urlstat
1084
1053
  UrlTrends Ranking Updater
1085
- URLy\ Warning
1054
+ URLy Warning
1086
1055
  URLy\.Warning
1087
1056
  Vacuum
1088
1057
  Vagabondo
1089
- VB\ Project
1058
+ VB Project
1090
1059
  vBSEO
1091
1060
  VCI
1092
1061
  via ggpht\.com GoogleImageProxy
@@ -1098,12 +1067,11 @@ VoidEYE
1098
1067
  Voil
1099
1068
  voltron
1100
1069
  voyager\/
1101
- VSAgent\/[0-9]
1102
- VSB-TUO\/[0-9]
1070
+ VSAgent\/
1071
+ VSB-TUO\/
1103
1072
  Vulnbusters Meter
1104
1073
  VYU2
1105
1074
  w3af\.org
1106
- W3C_I18n-Checker
1107
1075
  W3C_Unicorn
1108
1076
  W3C-checklink
1109
1077
  W3C-mobileOK
@@ -1116,16 +1084,15 @@ WatchMouse
1116
1084
  WbSrch\/
1117
1085
  WDT\.io
1118
1086
  web-capture\.net
1119
- Web-Monitoring
1120
1087
  Web-sniffer
1121
- Web\ Auto
1122
- Web\ Collage
1123
- Web\ Enhancer
1124
- Web\ Fetch
1125
- Web\ Fuck
1126
- Web\ Pix
1127
- Web\ Sauger
1128
- Web\ Sucker
1088
+ Web Auto
1089
+ Web Collage
1090
+ Web Enhancer
1091
+ Web Fetch
1092
+ Web Fuck
1093
+ Web Pix
1094
+ Web Sauger
1095
+ Web Sucker
1129
1096
  Webalta
1130
1097
  Webauskunft
1131
1098
  WebAuto
@@ -1135,36 +1102,35 @@ webcollage
1135
1102
  WebCookies
1136
1103
  WebCopier
1137
1104
  WebCorp
1138
- WebDataStats\/[0-9]
1105
+ WebDataStats
1139
1106
  WebDoc
1140
1107
  WebEnhancer
1141
1108
  WebFetch
1142
1109
  WebFuck
1143
- WebGo\ IS
1110
+ WebGazer
1111
+ WebGo IS
1144
1112
  WebImageCollector
1145
1113
  WebImages
1146
1114
  WebIndex
1147
1115
  webkit2png
1148
1116
  WebLeacher
1149
1117
  webmastercoffee
1150
- webmon
1118
+ webmon\s
1151
1119
  WebPix
1152
1120
  WebReaper
1153
1121
  WebSauger
1154
1122
  webscreenie
1155
1123
  Webshag
1156
1124
  Webshot
1157
- Website Analyzer\/
1158
- Website\ Quester
1159
- WebsiteExtractor
1125
+ Website Quester
1160
1126
  websitepulse agent
1161
- websitepulse[+ ]checker
1162
1127
  WebsiteQuester
1163
- Websnapr\/
1128
+ Websnapr
1129
+ WebSniffer
1164
1130
  Webster
1165
1131
  WebStripper
1166
1132
  WebSucker
1167
- Webthumb\/[0-9]
1133
+ Webthumb\/
1168
1134
  WebThumbnail
1169
1135
  WebWhacker
1170
1136
  WebZIP
@@ -1206,13 +1172,12 @@ x09Mozilla
1206
1172
  x22Mozilla
1207
1173
  XaxisSemanticsClassifier
1208
1174
  Xenu Link Sleuth
1209
- XING-contenttabreceiver\/[0-9]
1175
+ XING-contenttabreceiver
1210
1176
  xpymep([0-9]?)\.exe
1211
1177
  Y!J-(ASR|BSC)
1212
1178
  Y\!J-BRW
1213
1179
  Yaanb
1214
1180
  yacy
1215
- Yahoo Ad monitoring
1216
1181
  Yahoo Link Preview
1217
1182
  YahooCacheSystem
1218
1183
  YahooYSMcm
@@ -1224,7 +1189,7 @@ Yo-yo
1224
1189
  Yoleo Consumer
1225
1190
  yoogliFetchAgent
1226
1191
  YottaaMonitor
1227
- Your-Website-Sucks\/[0-9]
1192
+ Your-Website-Sucks
1228
1193
  yourls\.org
1229
1194
  YoYs\.net
1230
1195
  YP\.PL
@@ -1242,7 +1207,7 @@ ZnajdzFoto
1242
1207
  Zombie\.js
1243
1208
  Zoom\.Mac
1244
1209
  ZyBorg
1245
- [a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron)
1210
+ [a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
1246
1211
  ].strip.split(/\n+/).freeze
1247
1212
  end
1248
1213
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module CrawlerDetect
4
- VERSION = "0.1.5"
4
+ VERSION = "0.1.6"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawler_detect
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pavel Kozlov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-12-31 00:00:00.000000000 Z
11
+ date: 2019-02-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler