crawler_detect 1.2.3 → 1.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +1 -0
- data/.github/workflows/build.yml +6 -12
- data/.gitignore +53 -6
- data/.rubocop.yml +2 -1
- data/Dockerfile +20 -0
- data/README.md +13 -0
- data/crawler_detect.gemspec +10 -10
- data/docker-compose.yml +37 -0
- data/lib/crawler_detect/detector.rb +2 -4
- data/lib/crawler_detect/library/raw/Crawlers.json +1 -1
- data/lib/crawler_detect/version.rb +1 -1
- data/lib/rack/crawler_detect.rb +1 -1
- data/rspec-notes +33 -0
- metadata +32 -57
- data/Gemfile.lock +0 -117
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 219f45abe5e0918bed7a4cb308570f90fbd612542bf124a4fe4dcb792e9b592c
|
|
4
|
+
data.tar.gz: 387c421140d47482e9635e7100a0eb08041cb1dfeaa6ba4f6c7fc7098e712ef0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 35992b245adccc1c52538648ef0062d0e8585beff1a9b51f11cb3db71443b4687bb3878d249d058a6b17a2d624425947876bb7995f6357095866f56e700c599f
|
|
7
|
+
data.tar.gz: e2f2fb57f2d79bc28b560b132a6ddd7d5b37f22cc9f059367779589f486b858405dc2ab9e9d6e48f27ca196276d1e8d2bd29c585c0fa17a9316d1854d99cdbbc
|
data/.dockerignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Gemfile.lock
|
data/.github/workflows/build.yml
CHANGED
|
@@ -6,14 +6,11 @@ jobs:
|
|
|
6
6
|
lint:
|
|
7
7
|
runs-on: ubuntu-latest
|
|
8
8
|
steps:
|
|
9
|
-
- uses: actions/checkout@
|
|
9
|
+
- uses: actions/checkout@v4
|
|
10
10
|
- uses: ruby/setup-ruby@v1
|
|
11
11
|
with:
|
|
12
|
-
ruby-version:
|
|
13
|
-
|
|
14
|
-
run: |
|
|
15
|
-
gem install bundler
|
|
16
|
-
bundle install --jobs 4 --retry 3
|
|
12
|
+
ruby-version: '3.0'
|
|
13
|
+
bundler-cache: true
|
|
17
14
|
- run: bundle exec rubocop
|
|
18
15
|
test:
|
|
19
16
|
needs: lint
|
|
@@ -21,18 +18,15 @@ jobs:
|
|
|
21
18
|
strategy:
|
|
22
19
|
fail-fast: false
|
|
23
20
|
matrix:
|
|
24
|
-
ruby: [2.5, 2.6, 2.7, 3.0, 3.1]
|
|
21
|
+
ruby: [2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, jruby-9.3, jruby-head]
|
|
25
22
|
test-group: [1, 2, 3, 4]
|
|
26
23
|
name: Ruby ${{ matrix.ruby }}, test-group ${{ matrix.test-group }}
|
|
27
24
|
steps:
|
|
28
|
-
- uses: actions/checkout@
|
|
25
|
+
- uses: actions/checkout@v4
|
|
29
26
|
- uses: ruby/setup-ruby@v1
|
|
30
27
|
with:
|
|
31
28
|
ruby-version: ${{ matrix.ruby }}
|
|
32
|
-
|
|
33
|
-
run: |
|
|
34
|
-
gem install bundler
|
|
35
|
-
bundle install --jobs 4 --retry 3
|
|
29
|
+
bundler-cache: true
|
|
36
30
|
- name: test
|
|
37
31
|
run: |
|
|
38
32
|
bundle exec parallel_rspec spec/ -n 4 --only-group ${{ matrix.test-group }} --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
|
data/.gitignore
CHANGED
|
@@ -1,11 +1,58 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
*.gem
|
|
2
|
+
*.rbc
|
|
3
|
+
/.config
|
|
4
4
|
/coverage/
|
|
5
|
-
/
|
|
5
|
+
/InstalledFiles
|
|
6
6
|
/pkg/
|
|
7
7
|
/spec/reports/
|
|
8
|
+
/spec/examples.txt
|
|
9
|
+
/test/tmp/
|
|
10
|
+
/test/version_tmp/
|
|
8
11
|
/tmp/
|
|
9
12
|
|
|
10
|
-
#
|
|
11
|
-
.
|
|
13
|
+
# Used by dotenv library to load environment variables.
|
|
14
|
+
# .env
|
|
15
|
+
|
|
16
|
+
# Ignore Byebug command history file.
|
|
17
|
+
.byebug_history
|
|
18
|
+
|
|
19
|
+
## Specific to RubyMotion:
|
|
20
|
+
.dat*
|
|
21
|
+
.repl_history
|
|
22
|
+
build/
|
|
23
|
+
*.bridgesupport
|
|
24
|
+
build-iPhoneOS/
|
|
25
|
+
build-iPhoneSimulator/
|
|
26
|
+
|
|
27
|
+
## Specific to RubyMotion (use of CocoaPods):
|
|
28
|
+
#
|
|
29
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
|
30
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
|
31
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
|
32
|
+
#
|
|
33
|
+
# vendor/Pods/
|
|
34
|
+
|
|
35
|
+
## Documentation cache and generated files:
|
|
36
|
+
/.yardoc/
|
|
37
|
+
/_yardoc/
|
|
38
|
+
/doc/
|
|
39
|
+
/rdoc/
|
|
40
|
+
|
|
41
|
+
## Environment normalization:
|
|
42
|
+
/.bundle/
|
|
43
|
+
/vendor/bundle
|
|
44
|
+
/lib/bundler/man/
|
|
45
|
+
|
|
46
|
+
# for a library or gem, you might want to ignore these files since the code is
|
|
47
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
48
|
+
Gemfile.lock
|
|
49
|
+
.ruby-version
|
|
50
|
+
.ruby-gemset
|
|
51
|
+
|
|
52
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
|
53
|
+
.rvmrc
|
|
54
|
+
|
|
55
|
+
# Used by RuboCop. Remote config files pulled in from inherit_from directive.
|
|
56
|
+
# .rubocop-https?--*
|
|
57
|
+
|
|
58
|
+
.idea
|
data/.rubocop.yml
CHANGED
data/Dockerfile
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
ARG RUBY_VERSION="2.5"
|
|
2
|
+
FROM ruby:$RUBY_VERSION-alpine
|
|
3
|
+
|
|
4
|
+
ARG BUNDLER_VERSION="2.3"
|
|
5
|
+
RUN gem install bundler -v $BUNDLER_VERSION
|
|
6
|
+
|
|
7
|
+
RUN apk add --no-cache --update --upgrade \
|
|
8
|
+
git make gcc libc-dev
|
|
9
|
+
|
|
10
|
+
WORKDIR /app
|
|
11
|
+
|
|
12
|
+
COPY crawler_detect.gemspec ./
|
|
13
|
+
COPY Gemfile ./
|
|
14
|
+
COPY lib/crawler_detect/version.rb ./lib/crawler_detect/
|
|
15
|
+
|
|
16
|
+
RUN bundle install
|
|
17
|
+
|
|
18
|
+
COPY . .
|
|
19
|
+
|
|
20
|
+
CMD ["bundle", "exec", "rubocop"]
|
data/README.md
CHANGED
|
@@ -17,6 +17,10 @@ Comparing with other popular bot-detection gems:
|
|
|
17
17
|
|
|
18
18
|
In order to remain up-to-date, this gem does not accept any crawler data updates – any PRs to edit the crawler data should be offered to the original [JayBizzle/CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect) project.
|
|
19
19
|
|
|
20
|
+
## Requirements
|
|
21
|
+
|
|
22
|
+
- Ruby: MRI 2.5+ or JRuby 9.3+.
|
|
23
|
+
|
|
20
24
|
## Installation
|
|
21
25
|
Add this line to your application's Gemfile:
|
|
22
26
|
|
|
@@ -69,5 +73,14 @@ end
|
|
|
69
73
|
```
|
|
70
74
|
Make sure that your files are correct JSON files.
|
|
71
75
|
Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information.
|
|
76
|
+
|
|
77
|
+
## Development
|
|
78
|
+
|
|
79
|
+
You can run `rubocop` \ `rspec` with any ruby version using docker like this:
|
|
80
|
+
```bash
|
|
81
|
+
docker build --build-arg RUBY_VERSION=3.3 --build-arg BUNDLER_VERSION=2.5 -t crawler_detect:3.3 .
|
|
82
|
+
docker run -it crawler_detect:3.3 bundle exec rspec
|
|
83
|
+
```
|
|
84
|
+
|
|
72
85
|
## License
|
|
73
86
|
MIT License
|
data/crawler_detect.gemspec
CHANGED
|
@@ -24,15 +24,15 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
25
25
|
spec.require_paths = ["lib"]
|
|
26
26
|
|
|
27
|
-
spec.add_dependency "qonfig", "
|
|
27
|
+
spec.add_dependency "qonfig", ">= 0.24"
|
|
28
28
|
|
|
29
|
-
spec.add_development_dependency "activesupport", "
|
|
30
|
-
spec.add_development_dependency "
|
|
31
|
-
spec.add_development_dependency "
|
|
32
|
-
spec.add_development_dependency "
|
|
33
|
-
spec.add_development_dependency "
|
|
34
|
-
spec.add_development_dependency "
|
|
35
|
-
spec.add_development_dependency "
|
|
36
|
-
|
|
37
|
-
spec.
|
|
29
|
+
spec.add_development_dependency "activesupport", ">= 6.0"
|
|
30
|
+
spec.add_development_dependency "fuubar", ">= 2.5"
|
|
31
|
+
spec.add_development_dependency "parallel_tests", ">= 3.0"
|
|
32
|
+
spec.add_development_dependency "rack-test", ">= 2.1"
|
|
33
|
+
spec.add_development_dependency "rake", ">= 13.1"
|
|
34
|
+
spec.add_development_dependency "rspec", ">= 3.13"
|
|
35
|
+
spec.add_development_dependency "armitage-rubocop", ">= 0.82"
|
|
36
|
+
|
|
37
|
+
spec.required_ruby_version = ">= 2.5.0"
|
|
38
38
|
end
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
version: "3"
|
|
2
|
+
services:
|
|
3
|
+
ruby2.5:
|
|
4
|
+
build:
|
|
5
|
+
context: ./
|
|
6
|
+
dockerfile: Dockerfile
|
|
7
|
+
args:
|
|
8
|
+
RUBY_VERSION: 2.5
|
|
9
|
+
BUNDLER_VERSION: 2.3
|
|
10
|
+
ruby2.6:
|
|
11
|
+
build:
|
|
12
|
+
context: ./
|
|
13
|
+
dockerfile: Dockerfile
|
|
14
|
+
args:
|
|
15
|
+
RUBY_VERSION: 2.6
|
|
16
|
+
BUNDLER_VERSION: 2.4
|
|
17
|
+
ruby2.7:
|
|
18
|
+
build:
|
|
19
|
+
context: ./
|
|
20
|
+
dockerfile: Dockerfile
|
|
21
|
+
args:
|
|
22
|
+
RUBY_VERSION: 2.7
|
|
23
|
+
BUNDLER_VERSION: 2.4
|
|
24
|
+
ruby3.0:
|
|
25
|
+
build:
|
|
26
|
+
context: ./
|
|
27
|
+
dockerfile: Dockerfile
|
|
28
|
+
args:
|
|
29
|
+
RUBY_VERSION: 3.0
|
|
30
|
+
BUNDLER_VERSION: 2.4
|
|
31
|
+
ruby3.3:
|
|
32
|
+
build:
|
|
33
|
+
context: ./
|
|
34
|
+
dockerfile: Dockerfile
|
|
35
|
+
args:
|
|
36
|
+
RUBY_VERSION: 3.3
|
|
37
|
+
BUNDLER_VERSION: 2.5
|
|
@@ -11,9 +11,7 @@ module CrawlerDetect
|
|
|
11
11
|
|
|
12
12
|
# @return [true, false] Is User-agent a crawler?
|
|
13
13
|
def is_crawler?
|
|
14
|
-
@is_crawler ||=
|
|
15
|
-
!completely_exclusion? && matches_crawler_list?
|
|
16
|
-
end
|
|
14
|
+
@is_crawler ||= !completely_exclusion? && matches_crawler_list?
|
|
17
15
|
end
|
|
18
16
|
|
|
19
17
|
# @return [String] The detected crawler name from RAW data
|
|
@@ -28,7 +26,7 @@ module CrawlerDetect
|
|
|
28
26
|
# @return [true, false] Is User-agent in white-list?
|
|
29
27
|
def completely_exclusion?
|
|
30
28
|
@user_agent.gsub!(exclusions_matcher, "")
|
|
31
|
-
@user_agent.strip.
|
|
29
|
+
@user_agent.strip.empty?
|
|
32
30
|
end
|
|
33
31
|
|
|
34
32
|
# @private
|
|
@@ -1 +1 @@
|
|
|
1
|
-
[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
|
|
1
|
+
[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^Amazon CloudFront","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookcatalog","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","GeedoProductSearch","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","link-check","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MicrosoftPreview","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","nbertaupete95","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probely\\.com","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","pulsetic\\.com","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
|
data/lib/rack/crawler_detect.rb
CHANGED
data/rspec-notes
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
1) CrawlerDetect::Detector black list Mozilla/5.0 AppleWebKit/537.36 Chrome/114.0.5735.179 Safari/537.36 Google-Ads-Conversions
|
|
2
|
+
is expected to equal true
|
|
3
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
4
|
+
|
|
5
|
+
expected true
|
|
6
|
+
got false
|
|
7
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
2) CrawlerDetect::Detector black list Amazon CloudFront
|
|
11
|
+
is expected to equal true
|
|
12
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
13
|
+
|
|
14
|
+
expected true
|
|
15
|
+
got false
|
|
16
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
3) CrawlerDetect::Detector black list link-check/5.2.0
|
|
20
|
+
is expected to equal true
|
|
21
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
22
|
+
|
|
23
|
+
expected true
|
|
24
|
+
got false
|
|
25
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
4) CrawlerDetect::Detector black list Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com is expected to equal true
|
|
29
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
30
|
+
|
|
31
|
+
expected true
|
|
32
|
+
got false
|
|
33
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
metadata
CHANGED
|
@@ -1,153 +1,125 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawler_detect
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.2.
|
|
4
|
+
version: 1.2.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pavel Kozlov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2024-03-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: qonfig
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "
|
|
17
|
+
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
19
|
version: '0.24'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
|
-
- - "
|
|
24
|
+
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0.24'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: activesupport
|
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
|
30
|
-
requirements:
|
|
31
|
-
- - "~>"
|
|
32
|
-
- !ruby/object:Gem::Version
|
|
33
|
-
version: 6.0.3
|
|
34
|
-
type: :development
|
|
35
|
-
prerelease: false
|
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
-
requirements:
|
|
38
|
-
- - "~>"
|
|
39
|
-
- !ruby/object:Gem::Version
|
|
40
|
-
version: 6.0.3
|
|
41
|
-
- !ruby/object:Gem::Dependency
|
|
42
|
-
name: bundler
|
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
|
44
30
|
requirements:
|
|
45
31
|
- - ">="
|
|
46
32
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '
|
|
33
|
+
version: '6.0'
|
|
48
34
|
type: :development
|
|
49
35
|
prerelease: false
|
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
37
|
requirements:
|
|
52
38
|
- - ">="
|
|
53
39
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '
|
|
40
|
+
version: '6.0'
|
|
55
41
|
- !ruby/object:Gem::Dependency
|
|
56
42
|
name: fuubar
|
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
|
58
44
|
requirements:
|
|
59
|
-
- - "
|
|
45
|
+
- - ">="
|
|
60
46
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '2.
|
|
47
|
+
version: '2.5'
|
|
62
48
|
type: :development
|
|
63
49
|
prerelease: false
|
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
51
|
requirements:
|
|
66
|
-
- - "
|
|
52
|
+
- - ">="
|
|
67
53
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '2.
|
|
54
|
+
version: '2.5'
|
|
69
55
|
- !ruby/object:Gem::Dependency
|
|
70
56
|
name: parallel_tests
|
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
|
72
58
|
requirements:
|
|
73
|
-
- - "
|
|
74
|
-
- !ruby/object:Gem::Version
|
|
75
|
-
version: '2.0'
|
|
76
|
-
type: :development
|
|
77
|
-
prerelease: false
|
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
-
requirements:
|
|
80
|
-
- - "~>"
|
|
81
|
-
- !ruby/object:Gem::Version
|
|
82
|
-
version: '2.0'
|
|
83
|
-
- !ruby/object:Gem::Dependency
|
|
84
|
-
name: pry-meta
|
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
|
86
|
-
requirements:
|
|
87
|
-
- - "~>"
|
|
59
|
+
- - ">="
|
|
88
60
|
- !ruby/object:Gem::Version
|
|
89
|
-
version:
|
|
61
|
+
version: '3.0'
|
|
90
62
|
type: :development
|
|
91
63
|
prerelease: false
|
|
92
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
65
|
requirements:
|
|
94
|
-
- - "
|
|
66
|
+
- - ">="
|
|
95
67
|
- !ruby/object:Gem::Version
|
|
96
|
-
version:
|
|
68
|
+
version: '3.0'
|
|
97
69
|
- !ruby/object:Gem::Dependency
|
|
98
70
|
name: rack-test
|
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
|
100
72
|
requirements:
|
|
101
|
-
- - "
|
|
73
|
+
- - ">="
|
|
102
74
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: '
|
|
75
|
+
version: '2.1'
|
|
104
76
|
type: :development
|
|
105
77
|
prerelease: false
|
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
107
79
|
requirements:
|
|
108
|
-
- - "
|
|
80
|
+
- - ">="
|
|
109
81
|
- !ruby/object:Gem::Version
|
|
110
|
-
version: '
|
|
82
|
+
version: '2.1'
|
|
111
83
|
- !ruby/object:Gem::Dependency
|
|
112
84
|
name: rake
|
|
113
85
|
requirement: !ruby/object:Gem::Requirement
|
|
114
86
|
requirements:
|
|
115
87
|
- - ">="
|
|
116
88
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
89
|
+
version: '13.1'
|
|
118
90
|
type: :development
|
|
119
91
|
prerelease: false
|
|
120
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
93
|
requirements:
|
|
122
94
|
- - ">="
|
|
123
95
|
- !ruby/object:Gem::Version
|
|
124
|
-
version: '
|
|
96
|
+
version: '13.1'
|
|
125
97
|
- !ruby/object:Gem::Dependency
|
|
126
98
|
name: rspec
|
|
127
99
|
requirement: !ruby/object:Gem::Requirement
|
|
128
100
|
requirements:
|
|
129
|
-
- - "
|
|
101
|
+
- - ">="
|
|
130
102
|
- !ruby/object:Gem::Version
|
|
131
|
-
version: '3.
|
|
103
|
+
version: '3.13'
|
|
132
104
|
type: :development
|
|
133
105
|
prerelease: false
|
|
134
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
135
107
|
requirements:
|
|
136
|
-
- - "
|
|
108
|
+
- - ">="
|
|
137
109
|
- !ruby/object:Gem::Version
|
|
138
|
-
version: '3.
|
|
110
|
+
version: '3.13'
|
|
139
111
|
- !ruby/object:Gem::Dependency
|
|
140
112
|
name: armitage-rubocop
|
|
141
113
|
requirement: !ruby/object:Gem::Requirement
|
|
142
114
|
requirements:
|
|
143
|
-
- -
|
|
115
|
+
- - ">="
|
|
144
116
|
- !ruby/object:Gem::Version
|
|
145
117
|
version: '0.82'
|
|
146
118
|
type: :development
|
|
147
119
|
prerelease: false
|
|
148
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
149
121
|
requirements:
|
|
150
|
-
- -
|
|
122
|
+
- - ">="
|
|
151
123
|
- !ruby/object:Gem::Version
|
|
152
124
|
version: '0.82'
|
|
153
125
|
description: CrawlerDetect is a library to detect bots/crawlers via the user agent
|
|
@@ -157,18 +129,20 @@ executables: []
|
|
|
157
129
|
extensions: []
|
|
158
130
|
extra_rdoc_files: []
|
|
159
131
|
files:
|
|
132
|
+
- ".dockerignore"
|
|
160
133
|
- ".github/workflows/build.yml"
|
|
161
134
|
- ".gitignore"
|
|
162
135
|
- ".rspec"
|
|
163
136
|
- ".rubocop.yml"
|
|
164
137
|
- CHANGELOG.md
|
|
138
|
+
- Dockerfile
|
|
165
139
|
- Gemfile
|
|
166
|
-
- Gemfile.lock
|
|
167
140
|
- LICENSE.txt
|
|
168
141
|
- README.md
|
|
169
142
|
- Rakefile
|
|
170
143
|
- bin/update_raw_files
|
|
171
144
|
- crawler_detect.gemspec
|
|
145
|
+
- docker-compose.yml
|
|
172
146
|
- lib/crawler_detect.rb
|
|
173
147
|
- lib/crawler_detect/config.rb
|
|
174
148
|
- lib/crawler_detect/detector.rb
|
|
@@ -182,6 +156,7 @@ files:
|
|
|
182
156
|
- lib/crawler_detect/library/raw/Headers.json
|
|
183
157
|
- lib/crawler_detect/version.rb
|
|
184
158
|
- lib/rack/crawler_detect.rb
|
|
159
|
+
- rspec-notes
|
|
185
160
|
homepage: https://github.com/loadkpi/crawler_detect
|
|
186
161
|
licenses:
|
|
187
162
|
- MIT
|
|
@@ -194,14 +169,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
194
169
|
requirements:
|
|
195
170
|
- - ">="
|
|
196
171
|
- !ruby/object:Gem::Version
|
|
197
|
-
version:
|
|
172
|
+
version: 2.5.0
|
|
198
173
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
174
|
requirements:
|
|
200
175
|
- - ">="
|
|
201
176
|
- !ruby/object:Gem::Version
|
|
202
177
|
version: '0'
|
|
203
178
|
requirements: []
|
|
204
|
-
rubygems_version: 3.
|
|
179
|
+
rubygems_version: 3.2.33
|
|
205
180
|
signing_key:
|
|
206
181
|
specification_version: 4
|
|
207
182
|
summary: 'CrawlerDetect: detect bots/crawlers'
|
data/Gemfile.lock
DELETED
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
crawler_detect (1.2.3)
|
|
5
|
-
qonfig (~> 0.24)
|
|
6
|
-
|
|
7
|
-
GEM
|
|
8
|
-
remote: https://rubygems.org/
|
|
9
|
-
specs:
|
|
10
|
-
activesupport (6.0.6)
|
|
11
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
12
|
-
i18n (>= 0.7, < 2)
|
|
13
|
-
minitest (~> 5.1)
|
|
14
|
-
tzinfo (~> 1.1)
|
|
15
|
-
zeitwerk (~> 2.2, >= 2.2.2)
|
|
16
|
-
armitage-rubocop (0.82.0)
|
|
17
|
-
rubocop (= 0.82.0)
|
|
18
|
-
rubocop-performance (= 1.5.2)
|
|
19
|
-
rubocop-rails (= 2.5.2)
|
|
20
|
-
rubocop-rake (= 0.5.1)
|
|
21
|
-
rubocop-rspec (= 1.38.1)
|
|
22
|
-
ast (2.4.2)
|
|
23
|
-
awesome_print (1.9.2)
|
|
24
|
-
byebug (11.1.3)
|
|
25
|
-
coderay (1.1.3)
|
|
26
|
-
concurrent-ruby (1.1.10)
|
|
27
|
-
diff-lcs (1.5.0)
|
|
28
|
-
fuubar (2.5.1)
|
|
29
|
-
rspec-core (~> 3.0)
|
|
30
|
-
ruby-progressbar (~> 1.4)
|
|
31
|
-
i18n (1.12.0)
|
|
32
|
-
concurrent-ruby (~> 1.0)
|
|
33
|
-
jaro_winkler (1.5.4)
|
|
34
|
-
method_source (1.0.0)
|
|
35
|
-
minitest (5.16.3)
|
|
36
|
-
parallel (1.22.1)
|
|
37
|
-
parallel_tests (2.32.0)
|
|
38
|
-
parallel
|
|
39
|
-
parser (3.1.3.0)
|
|
40
|
-
ast (~> 2.4.1)
|
|
41
|
-
pry (0.14.1)
|
|
42
|
-
coderay (~> 1.1)
|
|
43
|
-
method_source (~> 1.0)
|
|
44
|
-
pry-byebug (3.10.1)
|
|
45
|
-
byebug (~> 11.0)
|
|
46
|
-
pry (>= 0.13, < 0.15)
|
|
47
|
-
pry-meta (0.0.10)
|
|
48
|
-
awesome_print
|
|
49
|
-
pry
|
|
50
|
-
pry-byebug
|
|
51
|
-
pry-remote
|
|
52
|
-
pry-remote (0.1.8)
|
|
53
|
-
pry (~> 0.9)
|
|
54
|
-
slop (~> 3.0)
|
|
55
|
-
qonfig (0.28.0)
|
|
56
|
-
rack (2.2.4)
|
|
57
|
-
rack-test (1.1.0)
|
|
58
|
-
rack (>= 1.0, < 3)
|
|
59
|
-
rainbow (3.1.1)
|
|
60
|
-
rake (13.0.6)
|
|
61
|
-
rexml (3.2.5)
|
|
62
|
-
rspec (3.12.0)
|
|
63
|
-
rspec-core (~> 3.12.0)
|
|
64
|
-
rspec-expectations (~> 3.12.0)
|
|
65
|
-
rspec-mocks (~> 3.12.0)
|
|
66
|
-
rspec-core (3.12.0)
|
|
67
|
-
rspec-support (~> 3.12.0)
|
|
68
|
-
rspec-expectations (3.12.0)
|
|
69
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
70
|
-
rspec-support (~> 3.12.0)
|
|
71
|
-
rspec-mocks (3.12.0)
|
|
72
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
73
|
-
rspec-support (~> 3.12.0)
|
|
74
|
-
rspec-support (3.12.0)
|
|
75
|
-
rubocop (0.82.0)
|
|
76
|
-
jaro_winkler (~> 1.5.1)
|
|
77
|
-
parallel (~> 1.10)
|
|
78
|
-
parser (>= 2.7.0.1)
|
|
79
|
-
rainbow (>= 2.2.2, < 4.0)
|
|
80
|
-
rexml
|
|
81
|
-
ruby-progressbar (~> 1.7)
|
|
82
|
-
unicode-display_width (>= 1.4.0, < 2.0)
|
|
83
|
-
rubocop-performance (1.5.2)
|
|
84
|
-
rubocop (>= 0.71.0)
|
|
85
|
-
rubocop-rails (2.5.2)
|
|
86
|
-
activesupport
|
|
87
|
-
rack (>= 1.1)
|
|
88
|
-
rubocop (>= 0.72.0)
|
|
89
|
-
rubocop-rake (0.5.1)
|
|
90
|
-
rubocop
|
|
91
|
-
rubocop-rspec (1.38.1)
|
|
92
|
-
rubocop (>= 0.68.1)
|
|
93
|
-
ruby-progressbar (1.11.0)
|
|
94
|
-
slop (3.6.0)
|
|
95
|
-
thread_safe (0.3.6)
|
|
96
|
-
tzinfo (1.2.10)
|
|
97
|
-
thread_safe (~> 0.1)
|
|
98
|
-
unicode-display_width (1.8.0)
|
|
99
|
-
zeitwerk (2.6.6)
|
|
100
|
-
|
|
101
|
-
PLATFORMS
|
|
102
|
-
ruby
|
|
103
|
-
|
|
104
|
-
DEPENDENCIES
|
|
105
|
-
activesupport (~> 6.0.3)
|
|
106
|
-
armitage-rubocop (= 0.82)
|
|
107
|
-
bundler (>= 1.15)
|
|
108
|
-
crawler_detect!
|
|
109
|
-
fuubar (~> 2.0)
|
|
110
|
-
parallel_tests (~> 2.0)
|
|
111
|
-
pry-meta (~> 0.0.10)
|
|
112
|
-
rack-test (~> 1.1)
|
|
113
|
-
rake (>= 10.0)
|
|
114
|
-
rspec (~> 3.0)
|
|
115
|
-
|
|
116
|
-
BUNDLED WITH
|
|
117
|
-
2.3.21
|