crawler_detect 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97314f6307705c0123a25333340c7509138cd8991eb1946a52d66c99df08ddac
-  data.tar.gz: 135a67b8502348dde1c86dd0e17e894dceafeb61b8acb74487f3d6e1832954e0
+  metadata.gz: f6e03716868421dc64e5acb715b35599776fbe3951bbeb7e462892ed3a6a51d6
+  data.tar.gz: 10b00a68949482af5cb6064f538e0829f2bcb836fe5605da48b48f0af7bbad71
 SHA512:
-  metadata.gz: 1f6dc6ef31c358ce0a5e94469971021ba5a1e1c8ba9133836e839053e4717c32d94a0c3dbb2d828ada989192f70e85d899ad87b9b56f0e26ef33f08d45209112
-  data.tar.gz: ed8cf6c609fe3b7e3102d9a6d60358d9e80e19f69b1a8c4b218bc929642991389e40fecf48c5de69975d76373b1406b7a89e1c9ec5ccdf2dd727710eb3363524
+  metadata.gz: cb66a7c310ca791038ca199c68626d522f99cb4ddff57382b07ecb91b824e44292362351ccb71622051ece1d8c7a900019627fed731e46f5474507b8b20f6aaa
+  data.tar.gz: 3f4c2484f4155d975915dd21e1c62d6f8dd19ddda4023cff9530701cf628ddf94c10f5e097c3cb85ffbf3d71686bd91d9b9ac71da59cf47933549fd847d2c350

data/.travis.yml

@@ -8,9 +8,9 @@ env:
   - "TEST_GROUP=3"
   - "TEST_GROUP=4"
 rvm:
-  - 2.3
   - 2.5
   - 2.6
+  - 2.7
 before_install: gem install bundler
 script:
   - bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log

data/Gemfile.lock

@@ -0,0 +1,81 @@
+PATH
+  remote: .
+  specs:
+    crawler_detect (0.1.11)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.4.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    awesome_print (1.8.0)
+    byebug (11.1.2)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.6)
+    diff-lcs (1.3)
+    fuubar (2.5.0)
+      rspec-core (~> 3.0)
+      ruby-progressbar (~> 1.4)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.0)
+    parallel (1.19.1)
+    parallel_tests (2.32.0)
+      parallel
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    pry-meta (0.0.10)
+      awesome_print
+      pry
+      pry-byebug
+      pry-remote
+    pry-remote (0.1.8)
+      pry (~> 0.9)
+      slop (~> 3.0)
+    rack (2.2.2)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rake (13.0.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    ruby-progressbar (1.10.1)
+    slop (3.6.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 5.2.0)
+  bundler (>= 1.15)
+  crawler_detect!
+  fuubar (~> 2.0)
+  parallel_tests (~> 2.0)
+  pry-meta (~> 0.0.10)
+  rack-test (~> 1.1)
+  rake (>= 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.1.3

data/lib/crawler_detect/library/crawlers.rb

@@ -56,6 +56,7 @@ AddThis
 ADmantX
 AdminLabs
 adressendeutschland
+adreview\/
 adscanner
 Adstxtaggregator
 adstxt-worker
@@ -95,7 +96,6 @@ AportWorm\/
 AppBeat\/
 AppEngine-Google
 AppleSyndication
-AppStoreScraperZ
 Aprc\/[0-9]
 Arachmo
 arachnode
@@ -110,6 +110,7 @@ Asterias
 Astute
 asynchttp
 Attach
+attohttpc
 autocite
 AutomaticWPTester
 Autonomy
@@ -134,6 +135,7 @@ BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising
+Bewica-security-scan
 Bidtellect
 BigBozz
 Bigfoot
@@ -148,6 +150,7 @@ Black Hole
 Blackboard Safeassign
 BlackWidow
 BlockNote\.Net
+BlogBridge
 Bloglines
 Bloglovin
 BlogPulseLive
@@ -155,6 +158,7 @@ BlogSearch
 Blogtrottr
 BlowFish
 boitho\.com-dc
+Boost\.Beast
 BPImageWalker
 Braintree-Webhooks
 Branch Metrics API
@@ -205,6 +209,7 @@ clips\.ua\.ac\.be
 Cloud mapping
 CloudEndure
 CloudFlare-AlwaysOnline
+Cloudflare-Healthchecks
 Cloudinary
 cmcm\.com
 coccoc
@@ -247,6 +252,7 @@ Daum(oa)?[ \/][0-9]
 dBpoweramp
 ddline
 deeris
+delve\.ai
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
@@ -265,6 +271,7 @@ DNSPod-reporting
 docoloc
 Dolphin http client
 DomainAppender
+DomainLabz
 Donuts Content Explorer
 dotMailer content retrieval
 dotSemantic
@@ -278,6 +285,7 @@ Drupal \(\+http:\/\/drupal\.org\/\)
 DTS Agent
 dubaiindex
 DuplexWeb-Google
+DynatraceSynthetic
 EARTHCOM
 Easy-Thumb
 EasyDL
@@ -308,6 +316,7 @@ ExactSearch
 ExaleadCloudview
 Excel\/
 exif
+ExoRank
 Exploratodo
 Express WebPictures
 Extreme Picture Finder
@@ -342,6 +351,7 @@ Fetch API
 Fetch\/[0-9]
 Fever\/[0-9]
 FHscan
+Filestack
 Fimap
 findlink
 findthatfile
@@ -359,6 +369,7 @@ http:\/\/www.neomo.de\/
 free thumbnails
 Freeuploader
 Funnelback
+Fuzz Faster U Fool
 G-i-g-a-b-o-t
 g00g1e\.net
 ganarvisitas
@@ -379,6 +390,7 @@ GigablastOpenSource
 GIS-LABS
 github-camo
 github\.com
+Goldfire Server
 Go [\d\.]* package http
 Go http package
 Go-Ahead-Got-It
@@ -398,6 +410,7 @@ Google Page Speed Insights
 Google PP Default
 Google Search Console
 Google Web Preview
+Google-Ads-Overview
 Google-Adwords
 Google-Apps-Script
 Google-Calendar-Importer
@@ -407,6 +420,7 @@ Google-Publisher-Plugin
 Google-Read-Aloud
 Google-SearchByImage
 Google-Site-Verification
+Google-speakr
 Google-Structured-Data-Testing-Tool
 Google-Youtube-Links
 google-xrawler
@@ -416,7 +430,6 @@ GoogleProducer
 GoogleSites
 Google-Transparency-Report
 Gookey
-GoScraper
 GoSpotCheck
 gosquared-thumbnailer
 Gotit
@@ -450,6 +463,7 @@ HEADMasterSEO
 HeartRails_Capture
 help@dataminr\.com
 heritrix
+Hexometer
 historious
 hkedcity
 hledejLevne\.cz
@@ -596,6 +610,7 @@ Liferea\/
 Lightspeedsystems
 Lighthouse
 Likse
+limber\.io
 Link Valet
 link_thumbnailer
 LinkAlarm\/
@@ -615,6 +630,7 @@ livedoor ScreenShot
 LoadImpactRload
 localsearch-web
 LongURL API
+longurl-r-package
 looid\.com
 looksystems\.net
 ltx71
@@ -649,6 +665,7 @@ MemGator
 Metaspinner
 MetaURI
 MFC_Tear_Sample
+MicroMessenger\/
 Microsearch
 Microsoft Office
 Microsoft Outlook
@@ -679,6 +696,7 @@ Morfeus Fucking Scanner
 Morning Paper
 MovableType
 mowser
+Mr\.4x3 Powered
 Mrcgiguy
 MS Web Services Client Protocol
 MSFrontPage
@@ -748,7 +766,6 @@ oegp
 Offline Explorer
 Offline Navigator
 OgScrper
-og-scraper
 okhttp
 omgili
 OMSC
@@ -758,6 +775,7 @@ Openfind
 OpenLinkProfiler
 Openstat\/
 OpenVAS
+OPPO A33
 Optimizer
 Orbiter
 OrgProbe\/
@@ -772,6 +790,7 @@ Page Valet
 page_verifier
 page scorer
 page2rss
+PageFreezer
 PageGrabber
 PagePeeker
 PageScorer
@@ -825,6 +844,7 @@ PostmanRuntime
 PostPost
 postrank
 PowerPoint\/
+Prebid
 Priceonomics Analysis Engine
 PrintFriendly
 PritTorrent
@@ -839,8 +859,8 @@ PRTG Network Monitor
 pshtt, https scanning
 PTST
 PTST\/[0-9]+
-Pulsepoint XT3 web scraper
 Pump
+python-httpx
 Python-httplib2
 python-requests
 Python-urllib
@@ -881,13 +901,16 @@ Robozilla
 ROI Hunter
 RPT-HTTPClient
 RSSOwl
+RyowlEngine
 safe-agent-scanner
 SalesIntelligent
 Saleslift
 Sendsay\.Ru
 SauceNAO
 SBIder
+sc-downloader
 scalaj-http
+Scamadviser-Frontend
 scan\.lol
 ScanAlert
 Scoop
@@ -895,7 +918,6 @@ scooter
 ScoutJet
 ScoutURLMonitor
 ScrapeBox Page Scanner
-SimpleScraper
 Scrapy
 Screaming
 ScreenShotService
@@ -921,6 +943,7 @@ Seobility
 SEOCentro
 SeoCheck
 SEOkicks
+SEOlizer
 Seomoz
 SEOprofiler
 SEOsearch
@@ -934,6 +957,7 @@ Shelob
 Shodan
 Shoppimon
 ShopWiki
+shortURL lengthener
 ShortLinkTranslate
 shrinktheweb
 Sideqik
@@ -1013,7 +1037,6 @@ summify
 SuperHTTP
 Surphace Scout
 Suzuran
-SwiteScraper
 Symfony BrowserKit
 Symfony2 BrowserKit
 SynHttpClient-Built
@@ -1102,7 +1125,6 @@ VB Project
 vBSEO
 VCI
 via ggpht\.com GoogleImageProxy
-VidibleScraper
 Virusdie
 visionutils
 vkShare
@@ -1191,11 +1213,13 @@ WhereGoes\?
 Whibse
 WhoRunsCoinHive
 Whynder Magnet
+WinHttp-Autoproxy-Service
 Windows-RSS-Platform
 WinPodder
 wkhtmlto
 wmtips
 Woko
+Wolfram HTTPClient
 woorankreview
 Word\/
 WordPress\/
@@ -1254,7 +1278,7 @@ ZnHTTP
 Zombie\.js
 Zoom\.Mac
 ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
+[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
       ].strip.split(/\n+/).freeze
     end
   end

data/lib/crawler_detect/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CrawlerDetect
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawler_detect
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Pavel Kozlov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2020-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -134,6 +134,7 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile