RubyGems - crawler_detect - Versions diffs - 0.1.12 → 1.1.1 - Mend

crawler_detect 0.1.12 → 1.1.1

Files changed (23) hide show

checksums.yaml +4 -4
data/.github/workflows/build.yml +38 -0
data/.rubocop.yml +13 -168
data/CHANGELOG.md +36 -0
data/Gemfile.lock +42 -6
data/README.md +27 -12
data/bin/update_raw_files +10 -0
data/crawler_detect.gemspec +6 -3
data/lib/crawler_detect.rb +29 -7
data/lib/crawler_detect/config.rb +29 -0
data/lib/crawler_detect/detector.rb +27 -14
data/lib/crawler_detect/library.rb +9 -3
data/lib/crawler_detect/library/crawlers.rb +6 -1277
data/lib/crawler_detect/library/exclusions.rb +6 -50
data/lib/crawler_detect/library/headers.rb +6 -17
data/lib/crawler_detect/library/loader.rb +18 -0
data/lib/crawler_detect/library/raw/Crawlers.json +1 -0
data/lib/crawler_detect/library/raw/Exclusions.json +1 -0
data/lib/crawler_detect/library/raw/Headers.json +1 -0
data/lib/crawler_detect/version.rb +2 -1
data/lib/rack/crawler_detect.rb +20 -17
metadata +41 -6
data/.travis.yml +0 -16

@@ -0,0 +1,10 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
+wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
+wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt

data/crawler_detect.gemspec CHANGED

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "crawler_detect/version"
@@ -17,14 +17,16 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.add_development_dependency "activesupport", "~> 5.2.0"
+  spec.add_dependency "qonfig", "~> 0.24"
+  spec.add_development_dependency "activesupport", "~> 6.0.3"
   spec.add_development_dependency "bundler", ">= 1.15"
   spec.add_development_dependency "fuubar", "~> 2.0"
   spec.add_development_dependency "parallel_tests", "~> 2.0"
@@ -32,4 +34,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "armitage-rubocop", "0.82"
 end

data/lib/crawler_detect.rb CHANGED

@@ -1,24 +1,46 @@
 # frozen_string_literal: true
-require "crawler_detect/detector"
-require "crawler_detect/library"
-require "crawler_detect/library/crawlers"
-require "crawler_detect/library/exclusions"
-require "crawler_detect/library/headers"
-require "crawler_detect/version"
+require "json"
+require "qonfig"
-require "rack/crawler_detect"
+require_relative "crawler_detect/config"
+require_relative "crawler_detect/detector"
+require_relative "crawler_detect/library/loader"
+require_relative "crawler_detect/library/crawlers"
+require_relative "crawler_detect/library/exclusions"
+require_relative "crawler_detect/library/headers"
+require_relative "crawler_detect/library"
+require_relative "crawler_detect/version"
+require_relative "rack/crawler_detect"
+# @since 0.1.0
 module CrawlerDetect
   class << self
+    # @param user_agent [String] User-agent string to detect
+    # @return [CrawlerDetect::Detector] Instance of detector class
     def new(user_agent)
       detector(user_agent)
     end
+    # @param user_agent [String] User-agent string to detect
+    # @return [true, false] Is User-agent a crawler?
     def is_crawler?(user_agent)
       detector(user_agent).is_crawler?
     end
+    # @since 1.0.0
+    # @param config [Proc]
+    def setup!(&config)
+      @config = CrawlerDetect::Config.new(&config)
+      Library::DATA_CLASSES.each(&:reload_data)
+    end
+    # @since 1.0.0
+    # @return [CrawlerDetect::Config] Instance of configuration class
+    def config
+      @config ||= CrawlerDetect::Config.new
+    end
     private
     def detector(user_agent)

data/lib/crawler_detect/config.rb ADDED

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+module CrawlerDetect
+  # Configuration of CrawlerDetect
+  #
+  # @see settings
+  # @since 1.0.0
+  class Config < ::Qonfig::DataSet
+    CUR_PATH = File.dirname(File.expand_path(__FILE__)).freeze
+    RAW_PATH = File.join(CUR_PATH, "library/raw").freeze
+    RAW_CRAWLERS_PATH   = File.join(RAW_PATH, "Crawlers.json").freeze
+    RAW_EXCLUSIONS_PATH = File.join(RAW_PATH, "Exclusions.json").freeze
+    RAW_HEADERS_PATH    = File.join(RAW_PATH, "Headers.json").freeze
+    # @return [String] path to crawlers raw JSON file
+    setting :raw_crawlers_path, RAW_CRAWLERS_PATH
+    # @return [String] path to exclusions raw JSON file
+    setting :raw_exclusions_path, RAW_EXCLUSIONS_PATH
+    # @return [String] path to headers raw JSON file
+    setting :raw_headers_path, RAW_HEADERS_PATH
+    validate :raw_crawlers_path,   :string, strict: true
+    validate :raw_exclusions_path, :string, strict: true
+    validate :raw_headers_path,    :string, strict: true
+  end
+end

data/lib/crawler_detect/detector.rb CHANGED

@@ -1,17 +1,22 @@
 # frozen_string_literal: true
 module CrawlerDetect
+  # since 0.1.0
   class Detector
+    # @param user_agent [String] User-agent string to detect
+    # @return [CrawlerDetect::Detector] instance of detector class
     def initialize(user_agent)
       @user_agent = user_agent.to_s.dup
     end
+    # @return [true, false] Is User-agent a crawler?
     def is_crawler?
       @is_crawler ||= begin
         !completely_exclusion? && matches_crawler_list?
       end
     end
+    # @return [String] The detected crawler name from RAW data
     def crawler_name
       return unless is_crawler?
       @crawler_name
@@ -19,22 +24,30 @@ module CrawlerDetect
     private
-      def completely_exclusion?
-        @user_agent.gsub!(exclusions_matcher, "")
-        @user_agent.strip.length == 0
-      end
+    # @private
+    # @return [true, false] Is User-agent in white-list?
+    def completely_exclusion?
+      @user_agent.gsub!(exclusions_matcher, "")
+      @user_agent.strip.length.zero?
+    end
-      def matches_crawler_list?
-        @crawler_name = crawlers_matcher.match(@user_agent).to_s.strip
-        !@crawler_name.empty?
-      end
+    # @private
+    # @return [true, false] Is User-agent in black-list?
+    def matches_crawler_list?
+      @crawler_name = crawlers_matcher.match(@user_agent).to_s.strip
+      !@crawler_name.empty?
+    end
-      def exclusions_matcher
-        CrawlerDetect::Library.get_regexp("exclusions")
-      end
+    # @private
+    # @return [Regexp] White-list of User-agents
+    def exclusions_matcher
+      CrawlerDetect::Library.get_regexp("exclusions")
+    end
-      def crawlers_matcher
-        CrawlerDetect::Library.get_regexp("crawlers")
-      end
+    # @private
+    # @return [Regexp] Black-list of User-agents
+    def crawlers_matcher
+      CrawlerDetect::Library.get_regexp("crawlers")
+    end
   end
 end

data/lib/crawler_detect/library.rb CHANGED

@@ -1,16 +1,22 @@
 # frozen_string_literal: true
 module CrawlerDetect
+  # @since 0.1.0
   module Library
+    DATA_CLASSES = [Library::Headers, Library::Exclusions, Library::Crawlers].freeze
     class << self
+      # @param param [String] Name of raw data
+      # @return [Regexp]
       def get_regexp(param)
         data = get_array(param)
-        %r[#{data.join('|')}]i
+        %r{#{data.join('|')}}i
       end
+      # @param param [String] Name of raw data
+      # @return [Array]
       def get_array(param)
-        const_name = "CrawlerDetect::Library::#{param.capitalize}::#{param.upcase}"
-        const_get(const_name)
+        const_get("CrawlerDetect::Library::#{param.capitalize}").send(:data)
       end
     end
   end

data/lib/crawler_detect/library/crawlers.rb CHANGED

@@ -1,1285 +1,14 @@
 # frozen_string_literal: true
-# rubocop:disable Layout/TrailingWhitespace
 module CrawlerDetect
   module Library
+    # @since 0.1.0
     module Crawlers
-      CRAWLERS = %q[
-.*Java.*outbrain
- YLT
-^b0t$
-^bluefish
-^Calypso v\/
-^COMODO DCV
-^DangDang
-^DavClnt
-^FDM
-^git\/
-^Goose\/
-^Grabber
-^HTTPClient\/
-^Java\/
-^Jeode\/
-^Jetty\/
-^Mail\/
-^Mget
-^Microsoft URL Control
-^NG\/[0-9\.]
-^NING\/
-^PHP\/[0-9]
-^RMA\/
-^Ruby|Ruby\/[0-9]
-^VSE\/[0-9]
-^WordPress\.com
-^XRL\/[0-9]
-^ZmEu
-008\/
-13TABS
-192\.comAgent
-2ip\.ru
-404enemy
-7Siters
-80legs
-a\.pr-cy\.ru
-a3logics\.in
-A6-Indexer
-Abonti
-Aboundex
-aboutthedomain
-Accoona-AI-Agent
-acoon
-acrylicapps\.com\/pulp
-Acunetix
-AdAuth\/
-adbeat
-AddThis
-ADmantX
-AdminLabs
-adressendeutschland
-adreview\/
-adscanner
-Adstxtaggregator
-adstxt-worker
-adstxt\.com
-agentslug
-AHC
-aihit
-aiohttp\/
-Airmail
-akka-http\/
-akula\/
-alertra
-alexa site audit
-Alibaba\.Security\.Heimdall
-Alligator
-allloadin
-AllSubmitter
-alyze\.info
-amagit
-^Amazon Simple Notification Service Agent$
-Anarchie
-AndroidDownloadManager
-Anemone
-AngleSharp
-annotate_google
-Ant\.com
-Anturis Agent
-AnyEvent-HTTP\/
-Apache Droid
-Apache OpenOffice
-Apache-HttpAsyncClient
-Apache-HttpClient
-ApacheBench
-Apexoo
-APIs-Google
-AportWorm\/
-AppBeat\/
-AppEngine-Google
-AppleSyndication
-Aprc\/[0-9]
-Arachmo
-arachnode
-Arachnophilia
-aria2
-Arukereso
-asafaweb
-AskQuickly
-Ask Jeeves
-ASPSeek
-Asterias
-Astute
-asynchttp
-Attach
-attohttpc
-autocite
-AutomaticWPTester
-Autonomy
-axios\/
-AWS Security Scanner
-B-l-i-t-z-B-O-T
-Backlink-Ceck
-backlink-check
-BacklinkHttpStatus
-BackStreet
-BackWeb
-Bad-Neighborhood
-Badass
-baidu\.com
-Bandit
-basicstate
-BatchFTP
-Battlezta Bazinga
-baypup\/
-BazQux
-BBBike
-BCKLINKS
-BDFetch
-BegunAdvertising
-Bewica-security-scan
-Bidtellect
-BigBozz
-Bigfoot
-biglotron
-BingLocalSearch
-BingPreview
-binlar
-biNu image cacher
-Bitacle
-biz_Directory
-Black Hole
-Blackboard Safeassign
-BlackWidow
-BlockNote\.Net
-BlogBridge
-Bloglines
-Bloglovin
-BlogPulseLive
-BlogSearch
-Blogtrottr
-BlowFish
-boitho\.com-dc
-Boost\.Beast
-BPImageWalker
-Braintree-Webhooks
-Branch Metrics API
-Branch-Passthrough
-Brandprotect
-BrandVerity
-Brandwatch
-Brodie\/
-Browsershots
-BUbiNG
-Buck\/
-Buddy
-BuiltWith
-Bullseye
-BunnySlippers
-Burf Search
-Butterfly\/
-BuzzSumo
-CAAM\/[0-9]
-CakePHP
-Calculon
-Canary%20Mail
-CaretNail
-catexplorador
-CC Metadata Scaper
-Cegbfeieh
-censys
-Cerberian Drtrs
-CERT\.at-Statistics-Survey
-cg-eye
-changedetection
-ChangesMeter
-Charlotte
-CheckHost
-checkprivacy
-CherryPicker
-ChinaClaw
-Chirp\/
-chkme\.com
-Chlooe
-Chromaxa
-CirrusExplorer
-CISPA Vulnerability Notification
-Citoid
-CJNetworkQuality
-Clarsentia
-clips\.ua\.ac\.be
-Cloud mapping
-CloudEndure
-CloudFlare-AlwaysOnline
-Cloudflare-Healthchecks
-Cloudinary
-cmcm\.com
-coccoc
-cognitiveseo
-colly -
-CommaFeed
-Commons-HttpClient
-commonscan
-contactbigdatafr
-contentkingapp
-convera
-CookieReports
-copyright sheriff
-CopyRightCheck
-Copyscape
-cortex\/
-Cosmos4j\.feedback
-Covario-IDS
-Craw\/
-Crescent
-Crowsnest
-Criteo
-CSHttp
-CSSCheck
-curb
-Curious George
-curl
-cuwhois\/
-cybo\.com
-DAP\/NetHTTP
-DareBoost
-DatabaseDriverMysqli
-DataCha0s
-Datafeedwatch
-Datanyze
-DataparkSearch
-dataprovider
-DataXu
-Daum(oa)?[ \/][0-9]
-dBpoweramp
-ddline
-deeris
-delve\.ai
-Demon
-DeuSu
-developers\.google\.com\/\+\/web\/snippet\/
-Devil
-Digg
-Digincore
-DigitalPebble
-Dirbuster
-Discourse Forum Onebox
-Disqus\/
-Dispatch\/
-DittoSpyder
-dlvr
-DMBrowser
-DNSPod-reporting
-docoloc
-Dolphin http client
-DomainAppender
-DomainLabz
-Donuts Content Explorer
-dotMailer content retrieval
-dotSemantic
-downforeveryoneorjustme
-Download Wonder
-downnotifier
-DowntimeDetector
-Drip
-drupact
-Drupal \(\+http:\/\/drupal\.org\/\)
-DTS Agent
-dubaiindex
-DuplexWeb-Google
-DynatraceSynthetic
-EARTHCOM
-Easy-Thumb
-EasyDL
-Ebingbong
-ec2linkfinder
-eCairn-Grabber
-eCatch
-ECCP
-eContext\/
-Ecxi
-EirGrabber
-ElectricMonk
-elefent
-EMail Exractor
-EMail Wolf
-EmailWolf
-Embarcadero
-Embed PHP Library
-Embedly
-endo\/
-europarchive\.org
-evc-batch
-EventMachine HttpClient
-Everwall Link Expander
-Evidon
-Evrinid
-ExactSearch
-ExaleadCloudview
-Excel\/
-exif
-ExoRank
-Exploratodo
-Express WebPictures
-Extreme Picture Finder
-EyeNetIE
-ezooms
-facebookexternalhit
-facebookexternalua
-facebookplatform
-fairshare
-Faraday v
-fasthttp
-Faveeo
-Favicon downloader
-faviconkit
-faviconarchive
-FavOrg
-Feed Wrangler
-Feedable\/
-Feedbin
-FeedBooster
-FeedBucket
-FeedBunch\/
-FeedBurner
-feeder
-Feedly
-FeedshowOnline
-Feedspot
-Feedwind\/
-FeedZcollector
-feeltiptop
-Fetch API
-Fetch\/[0-9]
-Fever\/[0-9]
-FHscan
-Filestack
-Fimap
-findlink
-findthatfile
-FlashGet
-FlipboardBrowserProxy
-FlipboardProxy
-FlipboardRSS
-Flock\/
-fluffy
-Flunky
-flynxapp
-forensiq
-FoundSeoTool
-http:\/\/www.neomo.de\/
-free thumbnails
-Freeuploader
-Funnelback
-Fuzz Faster U Fool
-G-i-g-a-b-o-t
-g00g1e\.net
-ganarvisitas
-geek-tools
-Genieo
-GentleSource
-GetCode
-Getintent
-GetLinkInfo
-getprismatic
-GetRight
-getroot
-GetURLInfo\/
-GetWeb
-Geziyor
-Ghost Inspector
-GigablastOpenSource
-GIS-LABS
-github-camo
-github\.com
-Goldfire Server
-Go [\d\.]* package http
-Go http package
-Go-Ahead-Got-It
-Go-http-client
-Go!Zilla
-gobyus
-gofetch
-GomezAgent
-gooblog
-Goodzer\/
-Google AppsViewer
-Google Desktop
-Google favicon
-Google Keyword Suggestion
-Google Keyword Tool
-Google Page Speed Insights
-Google PP Default
-Google Search Console
-Google Web Preview
-Google-Ads-Overview
-Google-Adwords
-Google-Apps-Script
-Google-Calendar-Importer
-Google-HotelAdsVerifier
-Google-HTTP-Java-Client
-Google-Publisher-Plugin
-Google-Read-Aloud
-Google-SearchByImage
-Google-Site-Verification
-Google-speakr
-Google-Structured-Data-Testing-Tool
-Google-Youtube-Links
-google-xrawler
-GoogleDocs
-GoogleHC\/
-GoogleProducer
-GoogleSites
-Google-Transparency-Report
-Gookey
-GoSpotCheck
-gosquared-thumbnailer
-Gotit
-GoZilla
-grabify
-GrabNet
-Grafula
-Grammarly
-GrapeFX
-GreatNews
-Gregarius
-GRequests
-grokkit
-grouphigh
-grub-client
-gSOAP\/
-GT::WWW
-GTmetrix
-GuzzleHttp
-gvfs\/
-HAA(A)?RTLAND http client
-Haansoft
-hackney\/
-Hadi Agent
-HappyApps-WebCheck
-Hatena
-Havij
-HaxerMen
-HeadlessChrome
-HEADMasterSEO
-HeartRails_Capture
-help@dataminr\.com
-heritrix
-Hexometer
-historious
-hkedcity
-hledejLevne\.cz
-Hloader
-HMView
-Holmes
-HonesoSearchEngine
-HootSuite Image proxy
-Hootsuite-WebFeed
-hosterstats
-HostTracker
-ht:\/\/check
-htdig
-HTMLparser
-htmlyse
-HTTP Banner Detection
-HTTP_Compression_Test
-http_request2
-http_requester
-http-get
-HTTP-Header-Abfrage
-http-kit
-http-request\/
-HTTP-Tiny
-HTTP::Lite
-http\.rb\/
-http_get
-HttpComponents
-httphr
-HTTPMon
-HTTPie
-httpRequest
-httpscheck
-httpssites_power
-httpunit
-HttpUrlConnection
-httrack
-huaweisymantec
-HubSpot
-Humanlinks
-i2kconnect\/
-Iblog
-ichiro
-Id-search
-IdeelaborPlagiaat
-IDG Twitter Links Resolver
-IDwhois\/
-Iframely
-igdeSpyder
-IlTrovatore
-Image Fetch
-Image Sucker
-ImageEngine\/
-ImageVisu\/
-Imagga
-imagineeasy
-imgsizer
-InAGist
-inbound\.li parser
-InDesign%20CC
-Indy Library
-InetURL
-infegy
-infohelfer
-InfoTekies
-InfoWizards Reciprocal Link
-inpwrd\.com
-instabid
-Instapaper
-Integrity
-integromedb
-Intelliseek
-InterGET
-internet_archive
-Internet Ninja
-InternetSeer
-internetVista monitor
-internetwache
-intraVnews
-IODC
-IOI
-iplabel
-ips-agent
-IPS\/[0-9]
-IPWorks HTTP\/S Component
-iqdb\/
-Iria
-Irokez
-isitup\.org
-iskanie
-isUp\.li
-iThemes Sync\/
-IZaBEE
-iZSearch
-JAHHO
-janforman
-Jaunt\/
-Jbrofuzz
-Jersey\/
-JetCar
-Jigsaw
-Jobboerse
-JobFeed discovery
-Jobg8 URL Monitor
-jobo
-Jobrapido
-Jobsearch1\.5
-JoinVision Generic
-JolokiaPwn
-Joomla
-Jorgee
-JS-Kit
-JustView
-Kaspersky Lab CFR link resolver
-Kelny\/
-Kerrigan\/
-KeyCDN
-Keyword Density
-Keywords Research
-khttp\/
-KickFire
-KimonoLabs\/
-Kml-Google
-knows\.is
-KOCMOHABT
-kouio
-kubectl
-kube-probe
-kulturarw3
-KumKie
-L\.webis
-Larbin
-Lavf\/
-LeechFTP
-LeechGet
-letsencrypt
-Lftp
-LibVLC
-LibWeb
-Libwhisker
-libwww
-Licorne
-Liferea\/
-Lightspeedsystems
-Lighthouse
-Likse
-limber\.io
-Link Valet
-link_thumbnailer
-LinkAlarm\/
-linkCheck
-linkdex
-LinkExaminer
-linkfluence
-linkpeek
-LinkPreviewGenerator
-LinkScan
-LinksManager
-LinkTiger
-LinkWalker
-Lipperhey
-Litemage_walker
-livedoor ScreenShot
-LoadImpactRload
-localsearch-web
-LongURL API
-longurl-r-package
-looid\.com
-looksystems\.net
-ltx71
-lua-resty-http
-lwp-request
-lwp-trivial
-LWP::Simple
-lycos
-LYT\.SR
-mabontland
-Mag-Net
-MagpieRSS
-Mail\.Ru
-MailChimp
-Majestic12
-makecontact\/
-Mandrill
-MapperCmd
-marketinggrader
-MarkMonitor
-MarkWatch
-Mass Downloader
-masscan\/
-Mata Hari
-Mediametric
-Mediapartners-Google
-mediawords
-MegaIndex\.ru
-MeltwaterNews
-Melvil Rawi
-MemGator
-Metaspinner
-MetaURI
-MFC_Tear_Sample
-MicroMessenger\/
-Microsearch
-Microsoft Office
-Microsoft Outlook
-Microsoft Windows Network Diagnostics
-Microsoft-WebDAV-MiniRedir
-Microsoft Data Access
-MIDown tool
-MIIxpc
-Mindjet
-Miniature\.io
-Miniflux
-Mister PiX
-mixdata dot com
-mixed-content-scan
-Mixmax-LinkPreview
-mixnode
-Mnogosearch
-mogimogi
-Mojeek
-Mojolicious \(Perl\)
-Monit\/
-monitis
-Monitority\/
-montastic
-MonTools
-Moreover
-Morfeus Fucking Scanner
-Morning Paper
-MovableType
-mowser
-Mr\.4x3 Powered
-Mrcgiguy
-MS Web Services Client Protocol
-MSFrontPage
-mShots
-MuckRack\/
-muhstik-scan
-MVAClient
-MxToolbox\/
-nagios
-Najdi\.si
-Name Intelligence
-Nameprotect
-Navroad
-NearSite
-Needle
-Nessus
-Net Vampire
-NetAnts
-NETCRAFT
-NetLyzer
-NetMechanic
-NetNewsWire
-Netpursual
-netresearch
-NetShelter ContentScan
-Netsparker
-NetTrack
-Netvibes
-NetZIP
-Neustar WPM
-NeutrinoAPI
-NewRelicPinger
-NewsBlur .*Finder
-NewsGator
-newsme
-newspaper\/
-NetSystemsResearch
-Nexgate Ruby Client
-NG-Search
-Nibbler
-NICErsPRO
-Nikto
-nineconnections
-NLNZ_IAHarvester
-Nmap Scripting Engine
-node-superagent
-node-urllib
-node\.io
-Nodemeter
-NodePing
-nominet\.org\.uk
-nominet\.uk
-Norton-Safeweb
-Notifixious
-notifyninja
-NotionEmbedder
-nuhk
-nutch
-Nuzzel
-nWormFeedFinder
-nyawc\/
-Nymesis
-NYU
-Ocelli\/
-Octopus
-oegp
-Offline Explorer
-Offline Navigator
-OgScrper
-okhttp
-omgili
-OMSC
-Online Domain Tools
-OpenCalaisSemanticProxy
-Openfind
-OpenLinkProfiler
-Openstat\/
-OpenVAS
-OPPO A33
-Optimizer
-Orbiter
-OrgProbe\/
-orion-semantics
-Outlook-Express
-Outlook-iOS
-ow\.ly
-Owler
-ownCloud News
-OxfordCloudService
-Page Valet
-page_verifier
-page scorer
-page2rss
-PageFreezer
-PageGrabber
-PagePeeker
-PageScorer
-Pagespeed\/
-Panopta
-panscient
-Papa Foto
-parsijoo
-Pavuk
-PayPal IPN
-pcBrowser
-Pcore-HTTP
-Pearltrees
-PECL::HTTP
-peerindex
-Peew
-PeoplePal
-Perlu -
-PhantomJS Screenshoter
-PhantomJS\/
-Photon\/
-phpservermon
-Pi-Monster
-Picscout
-Picsearch
-PictureFinder
-Pimonster
-ping\.blo\.gs
-Pingability
-PingAdmin\.Ru
-Pingdom
-Pingoscope
-PingSpot
-pinterest\.com
-Pixray
-Pizilla
-Plagger\/
-Ploetz \+ Zeller
-Plukkie
-plumanalytics
-PocketImageCache
-PocketParser
-Pockey
-POE-Component-Client-HTTP
-Polymail\/
-Pompos
-Porkbun
-Port Monitor
-postano
-PostmanRuntime
-PostPost
-postrank
-PowerPoint\/
-Prebid
-Priceonomics Analysis Engine
-PrintFriendly
-PritTorrent
-Prlog
-probethenet
-Project 25499
-prospectb2b
-Protopage
-ProWebWalker
-proximic
-PRTG Network Monitor
-pshtt, https scanning
-PTST
-PTST\/[0-9]+
-Pump
-python-httpx
-Python-httplib2
-python-requests
-Python-urllib
-Qirina Hurdler
-QQDownload
-QrafterPro
-Qseero
-Qualidator
-QueryN Metasearch
-queuedriver
-Quora Link Preview
-Qwantify
-Radian6
-RankActive
-RankFlex
-RankSonicSiteAuditor
-Re-re Studio
-ReactorNetty
-Readability
-RealDownload
-RealPlayer%20Downloader
-RebelMouse
-Recorder
-RecurPost\/
-redback\/
-ReederForMac
-Reeder\/
-ReGet
-RepoMonkey
-request\.js
-reqwest\/
-ResponseCodeTest
-RestSharp
-Riddler
-Rival IQ
-Robosourcer
-Robozilla
-ROI Hunter
-RPT-HTTPClient
-RSSOwl
-RyowlEngine
-safe-agent-scanner
-SalesIntelligent
-Saleslift
-Sendsay\.Ru
-SauceNAO
-SBIder
-sc-downloader
-scalaj-http
-Scamadviser-Frontend
-scan\.lol
-ScanAlert
-Scoop
-scooter
-ScoutJet
-ScoutURLMonitor
-ScrapeBox Page Scanner
-Scrapy
-Screaming
-ScreenShotService
-Scrubby
-Scrutiny\/
-search\.thunderstone
-Search37
-searchenginepromotionhelp
-Searchestate
-SearchExpress
-SearchSight
-Seeker
-semanticdiscovery
-semanticjuice
-Semiocast HTTP client
-Semrush
-sentry\/
-SEO Browser
-Seo Servis
-seo-nastroj\.cz
-seo4ajax
-Seobility
-SEOCentro
-SeoCheck
-SEOkicks
-SEOlizer
-Seomoz
-SEOprofiler
-SEOsearch
-seoscanners
-seositecheckup
-SEOstats
-servernfo
-sexsearcher
-Seznam
-Shelob
-Shodan
-Shoppimon
-ShopWiki
-shortURL lengthener
-ShortLinkTranslate
-shrinktheweb
-Sideqik
-SimplePie
-SimplyFast
-Siphon
-SISTRIX
-Site-Shot\/
-Site Sucker
-Site24x7
-SiteBar
-Sitebeam
-Sitebulb\/
-SiteCondor
-SiteExplorer
-SiteGuardian
-Siteimprove
-SiteIndexed
-Sitemap(s)? Generator
-SitemapGenerator
-SiteMonitor
-Siteshooter B0t
-SiteSnagger
-SiteSucker
-SiteTruth
-Sitevigil
-sitexy\.com
-SkypeUriPreview
-Slack\/
-slider\.com
-slurp
-SlySearch
-SmartDownload
-SMRF URL Expander
-SMUrlExpander
-Snake
-Snappy
-SnapSearch
-Snarfer\/
-SniffRSS
-sniptracker
-Snoopy
-SnowHaze Search
-sogou web
-SortSite
-Sottopop
-sovereign\.ai
-SpaceBison
-SpamExperts
-Spammen
-Spanner
-spaziodati
-SPDYCheck
-Specificfeeds
-speedy
-SPEng
-Spinn3r
-spray-can
-Sprinklr
-spyonweb
-sqlmap
-Sqlworm
-Sqworm
-SSL Labs
-ssl-tools
-StackRambler
-Statastico\/
-StatusCake
-Steeler
-Stratagems Kumo
-Stroke\.cz
-StudioFACA
-StumbleUpon
-suchen
-Sucuri
-summify
-SuperHTTP
-Surphace Scout
-Suzuran
-Symfony BrowserKit
-Symfony2 BrowserKit
-SynHttpClient-Built
-Sysomos
-sysscan
-Szukacz
-T0PHackTeam
-tAkeOut
-Tarantula\/
-Taringa UGC
-TarmotGezgin
-Teleport
-Telesoft
-Telesphoreo
-Telesphorep
-Tenon\.io
-teoma
-terrainformatica
-Test Certificate Info
-testuri
-Tetrahedron
-TextRazor Downloader
-The Drop Reaper
-The Expert HTML Source Viewer
-The Knowledge AI
-The Intraformant
-theinternetrules
-TheNomad
-Thinklab
-Thumbshots
-ThumbSniper
-Thumbor
-timewe\.net
-TinEye
-Tiny Tiny RSS
-TLSProbe\/
-Toata
-topster
-touche\.com
-Traackr\.com
-tracemyfile
-Trackuity
-TrapitAgent
-Trendiction
-Trendsmap
-trendspottr
-truwoGPS
-TryJsoup
-TulipChain
-Turingos
-Turnitin
-tweetedtimes
-Tweetminster
-Tweezler\/
-twibble
-Twice
-Twikle
-Twingly
-Twisted PageGetter
-Typhoeus
-ubermetrics-technologies
-uclassify
-UdmSearch
-unchaos
-unirest-java
-UniversalFeedParser
-Unshorten\.It
-Untiny
-UnwindFetchor
-updated
-updown\.io daemon
-Upflow
-Uptimia
-Urlcheckr
-URL Verifier
-URLitor
-urlresolver
-Urlstat
-URLTester
-UrlTrends Ranking Updater
-URLy Warning
-URLy\.Warning
-Vacuum
-Vagabondo
-VB Project
-vBSEO
-VCI
-via ggpht\.com GoogleImageProxy
-Virusdie
-visionutils
-vkShare
-VoidEYE
-Voil
-voltron
-voyager\/
-VSAgent\/
-VSB-TUO\/
-Vulnbusters Meter
-VYU2
-w3af\.org
-W3C_Unicorn
-W3C-checklink
-W3C-mobileOK
-WAC-OFU
-Wallpapers\/[0-9]+
-WallpapersHD
-wangling
-Wappalyzer
-WatchMouse
-WbSrch\/
-WDT\.io
-web-capture\.net
-Web-sniffer
-Web Auto
-Web Collage
-Web Enhancer
-Web Fetch
-Web Fuck
-Web Pix
-Web Sauger
-Web spyder
-Web Sucker
-Webalta
-Webauskunft
-WebAuto
-WebCapture
-WebClient\/
-webcollage
-WebCookies
-WebCopier
-WebCorp
-WebDataStats
-WebDoc
-WebEnhancer
-WebFetch
-WebFuck
-WebGazer
-WebGo IS
-WebImageCollector
-WebImages
-WebIndex
-webkit2png
-WebLeacher
-webmastercoffee
-webmon\s
-WebPix
-WebReaper
-WebSauger
-webscreenie
-Webshag
-Webshot
-Website Quester
-websitepulse agent
-WebsiteQuester
-Websnapr
-WebSniffer
-Webster
-WebStripper
-WebSucker
-Webthumb\/
-WebThumbnail
-WebWhacker
-WebZIP
-WeLikeLinks
-WEPA
-WeSEE
-wf84
-Wfuzz\/
-wget
-WhatsApp
-WhatsMyIP
-WhatWeb
-WhereGoes\?
-Whibse
-WhoRunsCoinHive
-Whynder Magnet
-WinHttp-Autoproxy-Service
-Windows-RSS-Platform
-WinPodder
-wkhtmlto
-wmtips
-Woko
-Wolfram HTTPClient
-woorankreview
-Word\/
-WordPress\/
-worldping-api
-WordupinfoSearch
-wotbox
-WP Engine Install Performance API
-wpif
-wprecon\.com survey
-WPScan
-wscheck
-Wtrace
-WWW-Collector-E
-WWW-Mechanize
-WWW::Document
-WWW::Mechanize
-www\.monitor\.us
-WWWOFFLE
-x09Mozilla
-x22Mozilla
-XaxisSemanticsClassifier
-Xenu Link Sleuth
-XING-contenttabreceiver
-xpymep([0-9]?)\.exe
-Y!J-(ASR|BSC)
-Y\!J-BRW
-Yaanb
-yacy
-Yahoo Link Preview
-YahooCacheSystem
-YahooYSMcm
-YandeG
-Yandex(?!Search)
-yanga
-yeti
-Yo-yo
-Yoleo Consumer
-yoogliFetchAgent
-YottaaMonitor
-Your-Website-Sucks
-yourls\.org
-YoYs\.net
-YP\.PL
-Zabbix
-Zade
-Zao
-Zauba
-Zemanta Aggregator
-Zend_Http_Client
-Zend\\\Http\\\Client
-Zermelo
-Zeus
-zgrab
-ZnajdzFoto
-ZnHTTP
-Zombie\.js
-Zoom\.Mac
-ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
-      ].strip.split(/\n+/).freeze
+      extend Loader
+      def self.data
+        @data ||= load_raw(CrawlerDetect.config.settings.raw_crawlers_path).freeze
+      end
     end
   end
 end