crawler_detect 1.2.3 → 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 337a710c949aeec626e9535171ea91d844b4b1ba28f4a5e3c997e5f752dcc24c
4
- data.tar.gz: 9988c3fe91abee0569fdec28d13ee4b7194d51fe7cb7146444bd228be4eb385b
3
+ metadata.gz: 219f45abe5e0918bed7a4cb308570f90fbd612542bf124a4fe4dcb792e9b592c
4
+ data.tar.gz: 387c421140d47482e9635e7100a0eb08041cb1dfeaa6ba4f6c7fc7098e712ef0
5
5
  SHA512:
6
- metadata.gz: e2e149dc13a4f1a09ebe4c020748e014d941969a738920a71d38c6999fe3200231900bb08530a6f90c6a613c394eac118159f68ca804b240ba485dfba47f0376
7
- data.tar.gz: 883b6547f9765b456cbf092ae386dfb1ab0faa286e756c833b059af9e8f55060bb3b37456bf5ace9c89d4d3dd4f3a943c28d72cbc0155ea486c4423b567da5d2
6
+ metadata.gz: 35992b245adccc1c52538648ef0062d0e8585beff1a9b51f11cb3db71443b4687bb3878d249d058a6b17a2d624425947876bb7995f6357095866f56e700c599f
7
+ data.tar.gz: e2f2fb57f2d79bc28b560b132a6ddd7d5b37f22cc9f059367779589f486b858405dc2ab9e9d6e48f27ca196276d1e8d2bd29c585c0fa17a9316d1854d99cdbbc
data/.dockerignore ADDED
@@ -0,0 +1 @@
1
+ Gemfile.lock
@@ -6,14 +6,11 @@ jobs:
6
6
  lint:
7
7
  runs-on: ubuntu-latest
8
8
  steps:
9
- - uses: actions/checkout@v2
9
+ - uses: actions/checkout@v4
10
10
  - uses: ruby/setup-ruby@v1
11
11
  with:
12
- ruby-version: 2.7
13
- - name: install gems
14
- run: |
15
- gem install bundler
16
- bundle install --jobs 4 --retry 3
12
+ ruby-version: '3.0'
13
+ bundler-cache: true
17
14
  - run: bundle exec rubocop
18
15
  test:
19
16
  needs: lint
@@ -21,18 +18,15 @@ jobs:
21
18
  strategy:
22
19
  fail-fast: false
23
20
  matrix:
24
- ruby: [2.5, 2.6, 2.7, 3.0, 3.1]
21
+ ruby: [2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, jruby-9.3, jruby-head]
25
22
  test-group: [1, 2, 3, 4]
26
23
  name: Ruby ${{ matrix.ruby }}, test-group ${{ matrix.test-group }}
27
24
  steps:
28
- - uses: actions/checkout@v2
25
+ - uses: actions/checkout@v4
29
26
  - uses: ruby/setup-ruby@v1
30
27
  with:
31
28
  ruby-version: ${{ matrix.ruby }}
32
- - name: install gems
33
- run: |
34
- gem install bundler
35
- bundle install --jobs 4 --retry 3
29
+ bundler-cache: true
36
30
  - name: test
37
31
  run: |
38
32
  bundle exec parallel_rspec spec/ -n 4 --only-group ${{ matrix.test-group }} --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
data/.gitignore CHANGED
@@ -1,11 +1,58 @@
1
- /.bundle/
2
- /.yardoc
3
- /_yardoc/
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
4
  /coverage/
5
- /doc/
5
+ /InstalledFiles
6
6
  /pkg/
7
7
  /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
8
11
  /tmp/
9
12
 
10
- # rspec failure tracking
11
- .rspec_status
13
+ # Used by dotenv library to load environment variables.
14
+ # .env
15
+
16
+ # Ignore Byebug command history file.
17
+ .byebug_history
18
+
19
+ ## Specific to RubyMotion:
20
+ .dat*
21
+ .repl_history
22
+ build/
23
+ *.bridgesupport
24
+ build-iPhoneOS/
25
+ build-iPhoneSimulator/
26
+
27
+ ## Specific to RubyMotion (use of CocoaPods):
28
+ #
29
+ # We recommend against adding the Pods directory to your .gitignore. However
30
+ # you should judge for yourself, the pros and cons are mentioned at:
31
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
32
+ #
33
+ # vendor/Pods/
34
+
35
+ ## Documentation cache and generated files:
36
+ /.yardoc/
37
+ /_yardoc/
38
+ /doc/
39
+ /rdoc/
40
+
41
+ ## Environment normalization:
42
+ /.bundle/
43
+ /vendor/bundle
44
+ /lib/bundler/man/
45
+
46
+ # for a library or gem, you might want to ignore these files since the code is
47
+ # intended to run in multiple environments; otherwise, check them in:
48
+ Gemfile.lock
49
+ .ruby-version
50
+ .ruby-gemset
51
+
52
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
53
+ .rvmrc
54
+
55
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
56
+ # .rubocop-https?--*
57
+
58
+ .idea
data/.rubocop.yml CHANGED
@@ -5,7 +5,8 @@ inherit_gem:
5
5
  - lib/rubocop.rspec.yml
6
6
 
7
7
  AllCops:
8
- TargetRubyVersion: 2.7.1
8
+ TargetRubyVersion: 2.5
9
+ NewCops: enable
9
10
  Include:
10
11
  - lib/**/*.rb
11
12
  - spec/**/*.rb
data/Dockerfile ADDED
@@ -0,0 +1,20 @@
1
+ ARG RUBY_VERSION="2.5"
2
+ FROM ruby:$RUBY_VERSION-alpine
3
+
4
+ ARG BUNDLER_VERSION="2.3"
5
+ RUN gem install bundler -v $BUNDLER_VERSION
6
+
7
+ RUN apk add --no-cache --update --upgrade \
8
+ git make gcc libc-dev
9
+
10
+ WORKDIR /app
11
+
12
+ COPY crawler_detect.gemspec ./
13
+ COPY Gemfile ./
14
+ COPY lib/crawler_detect/version.rb ./lib/crawler_detect/
15
+
16
+ RUN bundle install
17
+
18
+ COPY . .
19
+
20
+ CMD ["bundle", "exec", "rubocop"]
data/README.md CHANGED
@@ -17,6 +17,10 @@ Comparing with other popular bot-detection gems:
17
17
 
18
18
  In order to remain up-to-date, this gem does not accept any crawler data updates – any PRs to edit the crawler data should be offered to the original [JayBizzle/CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect) project.
19
19
 
20
+ ## Requirements
21
+
22
+ - Ruby: MRI 2.5+ or JRuby 9.3+.
23
+
20
24
  ## Installation
21
25
  Add this line to your application's Gemfile:
22
26
 
@@ -69,5 +73,14 @@ end
69
73
  ```
70
74
  Make sure that your files are correct JSON files.
71
75
  Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information.
76
+
77
+ ## Development
78
+
79
+ You can run `rubocop` \ `rspec` with any ruby version using docker like this:
80
+ ```bash
81
+ docker build --build-arg RUBY_VERSION=3.3 --build-arg BUNDLER_VERSION=2.5 -t crawler_detect:3.3 .
82
+ docker run -it crawler_detect:3.3 bundle exec rspec
83
+ ```
84
+
72
85
  ## License
73
86
  MIT License
@@ -24,15 +24,15 @@ Gem::Specification.new do |spec|
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_dependency "qonfig", "~> 0.24"
27
+ spec.add_dependency "qonfig", ">= 0.24"
28
28
 
29
- spec.add_development_dependency "activesupport", "~> 6.0.3"
30
- spec.add_development_dependency "bundler", ">= 1.15"
31
- spec.add_development_dependency "fuubar", "~> 2.0"
32
- spec.add_development_dependency "parallel_tests", "~> 2.0"
33
- spec.add_development_dependency "pry-meta", "~> 0.0.10"
34
- spec.add_development_dependency "rack-test", "~> 1.1"
35
- spec.add_development_dependency "rake", ">= 10.0"
36
- spec.add_development_dependency "rspec", "~> 3.0"
37
- spec.add_development_dependency "armitage-rubocop", "0.82"
29
+ spec.add_development_dependency "activesupport", ">= 6.0"
30
+ spec.add_development_dependency "fuubar", ">= 2.5"
31
+ spec.add_development_dependency "parallel_tests", ">= 3.0"
32
+ spec.add_development_dependency "rack-test", ">= 2.1"
33
+ spec.add_development_dependency "rake", ">= 13.1"
34
+ spec.add_development_dependency "rspec", ">= 3.13"
35
+ spec.add_development_dependency "armitage-rubocop", ">= 0.82"
36
+
37
+ spec.required_ruby_version = ">= 2.5.0"
38
38
  end
@@ -0,0 +1,37 @@
1
+ version: "3"
2
+ services:
3
+ ruby2.5:
4
+ build:
5
+ context: ./
6
+ dockerfile: Dockerfile
7
+ args:
8
+ RUBY_VERSION: 2.5
9
+ BUNDLER_VERSION: 2.3
10
+ ruby2.6:
11
+ build:
12
+ context: ./
13
+ dockerfile: Dockerfile
14
+ args:
15
+ RUBY_VERSION: 2.6
16
+ BUNDLER_VERSION: 2.4
17
+ ruby2.7:
18
+ build:
19
+ context: ./
20
+ dockerfile: Dockerfile
21
+ args:
22
+ RUBY_VERSION: 2.7
23
+ BUNDLER_VERSION: 2.4
24
+ ruby3.0:
25
+ build:
26
+ context: ./
27
+ dockerfile: Dockerfile
28
+ args:
29
+ RUBY_VERSION: 3.0
30
+ BUNDLER_VERSION: 2.4
31
+ ruby3.3:
32
+ build:
33
+ context: ./
34
+ dockerfile: Dockerfile
35
+ args:
36
+ RUBY_VERSION: 3.3
37
+ BUNDLER_VERSION: 2.5
@@ -11,9 +11,7 @@ module CrawlerDetect
11
11
 
12
12
  # @return [true, false] Is User-agent a crawler?
13
13
  def is_crawler?
14
- @is_crawler ||= begin
15
- !completely_exclusion? && matches_crawler_list?
16
- end
14
+ @is_crawler ||= !completely_exclusion? && matches_crawler_list?
17
15
  end
18
16
 
19
17
  # @return [String] The detected crawler name from RAW data
@@ -28,7 +26,7 @@ module CrawlerDetect
28
26
  # @return [true, false] Is User-agent in white-list?
29
27
  def completely_exclusion?
30
28
  @user_agent.gsub!(exclusions_matcher, "")
31
- @user_agent.strip.length.zero?
29
+ @user_agent.strip.empty?
32
30
  end
33
31
 
34
32
  # @private
@@ -1 +1 @@
1
- [" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
1
+ [" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^Amazon CloudFront","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookcatalog","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","GeedoProductSearch","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","link-check","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MicrosoftPreview","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","nbertaupete95","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probely\\.com","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","pulsetic\\.com","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
@@ -2,5 +2,5 @@
2
2
 
3
3
  # @since 0.1.0
4
4
  module CrawlerDetect
5
- VERSION = "1.2.3"
5
+ VERSION = "1.2.4"
6
6
  end
@@ -5,7 +5,7 @@ module Rack
5
5
  #
6
6
  # @since 0.1.0
7
7
  class CrawlerDetect
8
- def initialize(app, options = {})
8
+ def initialize(app, _options = {})
9
9
  Rack::Request::Helpers.module_eval do
10
10
  def is_crawler?
11
11
  env["rack.crawler_detect"][:is_crawler]
data/rspec-notes ADDED
@@ -0,0 +1,33 @@
1
+ 1) CrawlerDetect::Detector black list Mozilla/5.0 AppleWebKit/537.36 Chrome/114.0.5735.179 Safari/537.36 Google-Ads-Conversions
2
+ is expected to equal true
3
+ Failure/Error: it { is_expected.to be(true) }
4
+
5
+ expected true
6
+ got false
7
+ # ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
8
+
9
+
10
+ 2) CrawlerDetect::Detector black list Amazon CloudFront
11
+ is expected to equal true
12
+ Failure/Error: it { is_expected.to be(true) }
13
+
14
+ expected true
15
+ got false
16
+ # ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
17
+
18
+
19
+ 3) CrawlerDetect::Detector black list link-check/5.2.0
20
+ is expected to equal true
21
+ Failure/Error: it { is_expected.to be(true) }
22
+
23
+ expected true
24
+ got false
25
+ # ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
26
+
27
+
28
+ 4) CrawlerDetect::Detector black list Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com is expected to equal true
29
+ Failure/Error: it { is_expected.to be(true) }
30
+
31
+ expected true
32
+ got false
33
+ # ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
metadata CHANGED
@@ -1,153 +1,125 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawler_detect
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.3
4
+ version: 1.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pavel Kozlov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-12-18 00:00:00.000000000 Z
11
+ date: 2024-03-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: qonfig
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0.24'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - "~>"
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0.24'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: activesupport
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - "~>"
32
- - !ruby/object:Gem::Version
33
- version: 6.0.3
34
- type: :development
35
- prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - "~>"
39
- - !ruby/object:Gem::Version
40
- version: 6.0.3
41
- - !ruby/object:Gem::Dependency
42
- name: bundler
43
29
  requirement: !ruby/object:Gem::Requirement
44
30
  requirements:
45
31
  - - ">="
46
32
  - !ruby/object:Gem::Version
47
- version: '1.15'
33
+ version: '6.0'
48
34
  type: :development
49
35
  prerelease: false
50
36
  version_requirements: !ruby/object:Gem::Requirement
51
37
  requirements:
52
38
  - - ">="
53
39
  - !ruby/object:Gem::Version
54
- version: '1.15'
40
+ version: '6.0'
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: fuubar
57
43
  requirement: !ruby/object:Gem::Requirement
58
44
  requirements:
59
- - - "~>"
45
+ - - ">="
60
46
  - !ruby/object:Gem::Version
61
- version: '2.0'
47
+ version: '2.5'
62
48
  type: :development
63
49
  prerelease: false
64
50
  version_requirements: !ruby/object:Gem::Requirement
65
51
  requirements:
66
- - - "~>"
52
+ - - ">="
67
53
  - !ruby/object:Gem::Version
68
- version: '2.0'
54
+ version: '2.5'
69
55
  - !ruby/object:Gem::Dependency
70
56
  name: parallel_tests
71
57
  requirement: !ruby/object:Gem::Requirement
72
58
  requirements:
73
- - - "~>"
74
- - !ruby/object:Gem::Version
75
- version: '2.0'
76
- type: :development
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - "~>"
81
- - !ruby/object:Gem::Version
82
- version: '2.0'
83
- - !ruby/object:Gem::Dependency
84
- name: pry-meta
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - "~>"
59
+ - - ">="
88
60
  - !ruby/object:Gem::Version
89
- version: 0.0.10
61
+ version: '3.0'
90
62
  type: :development
91
63
  prerelease: false
92
64
  version_requirements: !ruby/object:Gem::Requirement
93
65
  requirements:
94
- - - "~>"
66
+ - - ">="
95
67
  - !ruby/object:Gem::Version
96
- version: 0.0.10
68
+ version: '3.0'
97
69
  - !ruby/object:Gem::Dependency
98
70
  name: rack-test
99
71
  requirement: !ruby/object:Gem::Requirement
100
72
  requirements:
101
- - - "~>"
73
+ - - ">="
102
74
  - !ruby/object:Gem::Version
103
- version: '1.1'
75
+ version: '2.1'
104
76
  type: :development
105
77
  prerelease: false
106
78
  version_requirements: !ruby/object:Gem::Requirement
107
79
  requirements:
108
- - - "~>"
80
+ - - ">="
109
81
  - !ruby/object:Gem::Version
110
- version: '1.1'
82
+ version: '2.1'
111
83
  - !ruby/object:Gem::Dependency
112
84
  name: rake
113
85
  requirement: !ruby/object:Gem::Requirement
114
86
  requirements:
115
87
  - - ">="
116
88
  - !ruby/object:Gem::Version
117
- version: '10.0'
89
+ version: '13.1'
118
90
  type: :development
119
91
  prerelease: false
120
92
  version_requirements: !ruby/object:Gem::Requirement
121
93
  requirements:
122
94
  - - ">="
123
95
  - !ruby/object:Gem::Version
124
- version: '10.0'
96
+ version: '13.1'
125
97
  - !ruby/object:Gem::Dependency
126
98
  name: rspec
127
99
  requirement: !ruby/object:Gem::Requirement
128
100
  requirements:
129
- - - "~>"
101
+ - - ">="
130
102
  - !ruby/object:Gem::Version
131
- version: '3.0'
103
+ version: '3.13'
132
104
  type: :development
133
105
  prerelease: false
134
106
  version_requirements: !ruby/object:Gem::Requirement
135
107
  requirements:
136
- - - "~>"
108
+ - - ">="
137
109
  - !ruby/object:Gem::Version
138
- version: '3.0'
110
+ version: '3.13'
139
111
  - !ruby/object:Gem::Dependency
140
112
  name: armitage-rubocop
141
113
  requirement: !ruby/object:Gem::Requirement
142
114
  requirements:
143
- - - '='
115
+ - - ">="
144
116
  - !ruby/object:Gem::Version
145
117
  version: '0.82'
146
118
  type: :development
147
119
  prerelease: false
148
120
  version_requirements: !ruby/object:Gem::Requirement
149
121
  requirements:
150
- - - '='
122
+ - - ">="
151
123
  - !ruby/object:Gem::Version
152
124
  version: '0.82'
153
125
  description: CrawlerDetect is a library to detect bots/crawlers via the user agent
@@ -157,18 +129,20 @@ executables: []
157
129
  extensions: []
158
130
  extra_rdoc_files: []
159
131
  files:
132
+ - ".dockerignore"
160
133
  - ".github/workflows/build.yml"
161
134
  - ".gitignore"
162
135
  - ".rspec"
163
136
  - ".rubocop.yml"
164
137
  - CHANGELOG.md
138
+ - Dockerfile
165
139
  - Gemfile
166
- - Gemfile.lock
167
140
  - LICENSE.txt
168
141
  - README.md
169
142
  - Rakefile
170
143
  - bin/update_raw_files
171
144
  - crawler_detect.gemspec
145
+ - docker-compose.yml
172
146
  - lib/crawler_detect.rb
173
147
  - lib/crawler_detect/config.rb
174
148
  - lib/crawler_detect/detector.rb
@@ -182,6 +156,7 @@ files:
182
156
  - lib/crawler_detect/library/raw/Headers.json
183
157
  - lib/crawler_detect/version.rb
184
158
  - lib/rack/crawler_detect.rb
159
+ - rspec-notes
185
160
  homepage: https://github.com/loadkpi/crawler_detect
186
161
  licenses:
187
162
  - MIT
@@ -194,14 +169,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
194
169
  requirements:
195
170
  - - ">="
196
171
  - !ruby/object:Gem::Version
197
- version: '0'
172
+ version: 2.5.0
198
173
  required_rubygems_version: !ruby/object:Gem::Requirement
199
174
  requirements:
200
175
  - - ">="
201
176
  - !ruby/object:Gem::Version
202
177
  version: '0'
203
178
  requirements: []
204
- rubygems_version: 3.4.1
179
+ rubygems_version: 3.2.33
205
180
  signing_key:
206
181
  specification_version: 4
207
182
  summary: 'CrawlerDetect: detect bots/crawlers'
data/Gemfile.lock DELETED
@@ -1,117 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- crawler_detect (1.2.3)
5
- qonfig (~> 0.24)
6
-
7
- GEM
8
- remote: https://rubygems.org/
9
- specs:
10
- activesupport (6.0.6)
11
- concurrent-ruby (~> 1.0, >= 1.0.2)
12
- i18n (>= 0.7, < 2)
13
- minitest (~> 5.1)
14
- tzinfo (~> 1.1)
15
- zeitwerk (~> 2.2, >= 2.2.2)
16
- armitage-rubocop (0.82.0)
17
- rubocop (= 0.82.0)
18
- rubocop-performance (= 1.5.2)
19
- rubocop-rails (= 2.5.2)
20
- rubocop-rake (= 0.5.1)
21
- rubocop-rspec (= 1.38.1)
22
- ast (2.4.2)
23
- awesome_print (1.9.2)
24
- byebug (11.1.3)
25
- coderay (1.1.3)
26
- concurrent-ruby (1.1.10)
27
- diff-lcs (1.5.0)
28
- fuubar (2.5.1)
29
- rspec-core (~> 3.0)
30
- ruby-progressbar (~> 1.4)
31
- i18n (1.12.0)
32
- concurrent-ruby (~> 1.0)
33
- jaro_winkler (1.5.4)
34
- method_source (1.0.0)
35
- minitest (5.16.3)
36
- parallel (1.22.1)
37
- parallel_tests (2.32.0)
38
- parallel
39
- parser (3.1.3.0)
40
- ast (~> 2.4.1)
41
- pry (0.14.1)
42
- coderay (~> 1.1)
43
- method_source (~> 1.0)
44
- pry-byebug (3.10.1)
45
- byebug (~> 11.0)
46
- pry (>= 0.13, < 0.15)
47
- pry-meta (0.0.10)
48
- awesome_print
49
- pry
50
- pry-byebug
51
- pry-remote
52
- pry-remote (0.1.8)
53
- pry (~> 0.9)
54
- slop (~> 3.0)
55
- qonfig (0.28.0)
56
- rack (2.2.4)
57
- rack-test (1.1.0)
58
- rack (>= 1.0, < 3)
59
- rainbow (3.1.1)
60
- rake (13.0.6)
61
- rexml (3.2.5)
62
- rspec (3.12.0)
63
- rspec-core (~> 3.12.0)
64
- rspec-expectations (~> 3.12.0)
65
- rspec-mocks (~> 3.12.0)
66
- rspec-core (3.12.0)
67
- rspec-support (~> 3.12.0)
68
- rspec-expectations (3.12.0)
69
- diff-lcs (>= 1.2.0, < 2.0)
70
- rspec-support (~> 3.12.0)
71
- rspec-mocks (3.12.0)
72
- diff-lcs (>= 1.2.0, < 2.0)
73
- rspec-support (~> 3.12.0)
74
- rspec-support (3.12.0)
75
- rubocop (0.82.0)
76
- jaro_winkler (~> 1.5.1)
77
- parallel (~> 1.10)
78
- parser (>= 2.7.0.1)
79
- rainbow (>= 2.2.2, < 4.0)
80
- rexml
81
- ruby-progressbar (~> 1.7)
82
- unicode-display_width (>= 1.4.0, < 2.0)
83
- rubocop-performance (1.5.2)
84
- rubocop (>= 0.71.0)
85
- rubocop-rails (2.5.2)
86
- activesupport
87
- rack (>= 1.1)
88
- rubocop (>= 0.72.0)
89
- rubocop-rake (0.5.1)
90
- rubocop
91
- rubocop-rspec (1.38.1)
92
- rubocop (>= 0.68.1)
93
- ruby-progressbar (1.11.0)
94
- slop (3.6.0)
95
- thread_safe (0.3.6)
96
- tzinfo (1.2.10)
97
- thread_safe (~> 0.1)
98
- unicode-display_width (1.8.0)
99
- zeitwerk (2.6.6)
100
-
101
- PLATFORMS
102
- ruby
103
-
104
- DEPENDENCIES
105
- activesupport (~> 6.0.3)
106
- armitage-rubocop (= 0.82)
107
- bundler (>= 1.15)
108
- crawler_detect!
109
- fuubar (~> 2.0)
110
- parallel_tests (~> 2.0)
111
- pry-meta (~> 0.0.10)
112
- rack-test (~> 1.1)
113
- rake (>= 10.0)
114
- rspec (~> 3.0)
115
-
116
- BUNDLED WITH
117
- 2.3.21