crawler_detect 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 619a1e011e85b433b0a8713e42b5074c065b0a153c1a05d9794b0ba8eb9657f3
-  data.tar.gz: 7ef04e254e61277a585acaca43cc867978b2a20a8b72e41729c57c5918b33eea
+  metadata.gz: bd0b64e074f079c213603a34e49b6e7079f96981c94107faeb55ffbae2a53b13
+  data.tar.gz: aba0f088f8d4412240a39048893f6b942fc7d2fe1e8cdaa7d8605539f3e5402a
 SHA512:
-  metadata.gz: 70c0923069f10f32a7c98f49625c203fee0fa353304011eaa560fc592fd50b3ed750d5eb2559d142fac80f4834be9b2cb44afdc690f6371b170685929578f31e
-  data.tar.gz: 5c5bcb0383ce46b067867c1414a1ac07b08ffa4309e4bea164bf77f9e4a63eaf127389e4c367c55ef357767e12759627bfceed1be447b8d82883070026d41b94
+  metadata.gz: b0c95712fa7ecd5a0202c8944716cd76a163fe4e257ac946157a2db1cbec2548d6def7223c26113723f68649f4a84a06568c7fd3f495fd4ab96e3c457a532b99
+  data.tar.gz: 2b2e43671c5977347e6f9b1b69a25bcff47fec1666c582b919e885f3c053223f537a795f1eac6d61b607cd2c7782d9f775fe243ad5e7c1831f5565e877bb7983

data/lib/crawler_detect/library/crawlers.rb

@@ -35,7 +35,6 @@ module CrawlerDetect
 ^PHP\/[0-9]
 ^RMA\/
 ^Ruby|Ruby\/[0-9]
-^scrutiny\/
 ^VSE\/[0-9]
 ^WordPress\.com
 ^XRL\/[0-9]
@@ -76,10 +75,12 @@ Anarchie
 AndroidDownloadManager
 Anemone
 AngleSharp\/
+annotate_google
 Ant\.com
 Anturis Agent
 AnyEvent-HTTP\/
 Apache Droid
+Apache OpenOffice
 Apache-HttpAsyncClient\/
 Apache-HttpClient\/
 ApacheBench\/
@@ -122,6 +123,7 @@ BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising\/
+Bidtellect\/
 BigBozz
 Bigfoot
 biglotron
@@ -134,6 +136,8 @@ biz_Directory
 Black\ Hole
 Blackboard Safeassign
 BlackWidow
+BlockNote\.Net
+Bloglines\/
 Bloglovin
 BlogPulseLive
 BlogSearch
@@ -219,10 +223,12 @@ cybo\.com
 DareBoost
 DatabaseDriverMysqli
 DataCha0s
+Datanyze
 DataparkSearch
 dataprovider
 DataXu
 Daum(oa)?[ \/][0-9]
+DemandbasePublisherAnalyzer\/
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
@@ -273,6 +279,7 @@ EMail\ Wolf
 EmailWolf
 Embed PHP Library
 Embedly
+endo\/
 europarchive\.org
 evc-batch
 EventMachine HttpClient
@@ -299,6 +306,7 @@ Faveeo
 Favicon downloader
 FavOrg
 Feed Wrangler
+Feedable\/
 Feedbin
 FeedBooster
 FeedBucket
@@ -306,8 +314,11 @@ FeedBunch\/[0-9]
 FeedBurner
 FeedChecker
 Feedly
+Feedreader
+FeedshowOnline
 Feedspot
 Feedwind\/[0-9]
+FeedZcollector
 feeltiptop
 Fetch API
 Fetch\/[0-9]
@@ -424,6 +435,7 @@ HostTracker
 ht:\/\/check
 htdig
 HTMLparser
+htmlyse\.com
 http-get
 HTTP-Header-Abfrage
 http-kit
@@ -457,6 +469,7 @@ IDwhois\/[0-9]
 Iframely
 igdeSpyder
 IlTrovatore
+ImageVisu\/
 Image\ Fetch
 Image\ Sucker
 ImageEngine\/
@@ -466,6 +479,7 @@ InAGist
 inbound\.li parser
 InDesign%20CC
 Indy\ Library
+InetURL
 infegy
 infohelfer
 InfoTekies
@@ -563,6 +577,7 @@ Lipperhey
 Litemage_walker
 livedoor ScreenShot
 LoadImpactRload
+localsearch-web
 LongURL API
 looksystems\.net
 ltx71
@@ -609,6 +624,7 @@ Miniflux
 Mister\ PiX
 mixdata dot com
 mixed-content-scan
+Mixmax-LinkPreview
 mixnode
 Mnogosearch
 mogimogi
@@ -645,6 +661,7 @@ NetAnts
 NETCRAFT
 NetLyzer
 NetMechanic
+NetNewsWire
 Netpursual
 netresearch
 NetShelter ContentScan
@@ -683,6 +700,7 @@ NYU
 Ocelli\/[0-9]
 Octopus
 oegp
+og-scraper\/
 Offline Explorer
 Offline\ Navigator
 okhttp
@@ -748,6 +766,7 @@ PingSpot
 pinterest\.com
 Pixray
 Pizilla
+Plagger\/
 PleaseCrawl
 Ploetz \+ Zeller
 Plukkie
@@ -795,6 +814,7 @@ Radian6
 RankActive
 RankFlex
 RankSonicSiteAuditor
+raynette_httprequest
 Readability
 RealDownload
 RealPlayer%20Downloader
@@ -816,6 +836,7 @@ Robozilla\/[0-9]
 ROI Hunter
 RPT-HTTPClient
 RSSOwl
+RssReader\/
 safe-agent-scanner
 SalesIntelligent
 Saleslift
@@ -824,6 +845,7 @@ SBIder
 scalaj-http
 scan\.lol
 ScanAlert
+ScrapeBox Page Scanner
 Scoop
 scooter
 ScoutJet
@@ -832,9 +854,11 @@ Scrapy
 Screaming
 ScreenShotService\/[0-9]
 Scrubby
+Scrutiny\/
 Search37\/
 search\.thunderstone
 Searchestate
+SearchExpress
 SearchSight
 Seeker
 semanticdiscovery
@@ -899,6 +923,7 @@ SlySearch
 SmartDownload
 SMRF URL Expander
 SMUrlExpander
+Snarfer\/
 Snake
 Snappy
 SniffRSS
@@ -998,6 +1023,7 @@ ubermetrics-technologies
 uclassify
 uCrawlr\/
 UdmSearch
+unirest-java
 UniversalFeedParser
 Unshorten\.It
 Untiny
@@ -1111,11 +1137,13 @@ WhoRunsCoinHive
 Whynder Magnet
 Windows-RSS-Platform
 WinHttpRequest
+WinPodder
 wkhtmlto
 wmtips
 Woko
 Word\/
 WordPress\/
+WordupinfoSearch
 wotbox
 WP Engine Install Performance API
 wpif

data/lib/crawler_detect/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CrawlerDetect
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawler_detect
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Pavel Kozlov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-23 00:00:00.000000000 Z
+date: 2018-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler