crawler_detect 1.2.3 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +1 -0
- data/.github/workflows/build.yml +6 -12
- data/.gitignore +53 -6
- data/.rubocop.yml +2 -1
- data/Dockerfile +20 -0
- data/README.md +13 -0
- data/crawler_detect.gemspec +10 -10
- data/docker-compose.yml +37 -0
- data/lib/crawler_detect/detector.rb +2 -4
- data/lib/crawler_detect/library/raw/Crawlers.json +1 -1
- data/lib/crawler_detect/version.rb +1 -1
- data/lib/rack/crawler_detect.rb +1 -1
- data/rspec-notes +33 -0
- metadata +32 -57
- data/Gemfile.lock +0 -117
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 219f45abe5e0918bed7a4cb308570f90fbd612542bf124a4fe4dcb792e9b592c
|
|
4
|
+
data.tar.gz: 387c421140d47482e9635e7100a0eb08041cb1dfeaa6ba4f6c7fc7098e712ef0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 35992b245adccc1c52538648ef0062d0e8585beff1a9b51f11cb3db71443b4687bb3878d249d058a6b17a2d624425947876bb7995f6357095866f56e700c599f
|
|
7
|
+
data.tar.gz: e2f2fb57f2d79bc28b560b132a6ddd7d5b37f22cc9f059367779589f486b858405dc2ab9e9d6e48f27ca196276d1e8d2bd29c585c0fa17a9316d1854d99cdbbc
|
data/.dockerignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Gemfile.lock
|
data/.github/workflows/build.yml
CHANGED
|
@@ -6,14 +6,11 @@ jobs:
|
|
|
6
6
|
lint:
|
|
7
7
|
runs-on: ubuntu-latest
|
|
8
8
|
steps:
|
|
9
|
-
- uses: actions/checkout@
|
|
9
|
+
- uses: actions/checkout@v4
|
|
10
10
|
- uses: ruby/setup-ruby@v1
|
|
11
11
|
with:
|
|
12
|
-
ruby-version:
|
|
13
|
-
|
|
14
|
-
run: |
|
|
15
|
-
gem install bundler
|
|
16
|
-
bundle install --jobs 4 --retry 3
|
|
12
|
+
ruby-version: '3.0'
|
|
13
|
+
bundler-cache: true
|
|
17
14
|
- run: bundle exec rubocop
|
|
18
15
|
test:
|
|
19
16
|
needs: lint
|
|
@@ -21,18 +18,15 @@ jobs:
|
|
|
21
18
|
strategy:
|
|
22
19
|
fail-fast: false
|
|
23
20
|
matrix:
|
|
24
|
-
ruby: [2.5, 2.6, 2.7, 3.0, 3.1]
|
|
21
|
+
ruby: [2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, jruby-9.3, jruby-head]
|
|
25
22
|
test-group: [1, 2, 3, 4]
|
|
26
23
|
name: Ruby ${{ matrix.ruby }}, test-group ${{ matrix.test-group }}
|
|
27
24
|
steps:
|
|
28
|
-
- uses: actions/checkout@
|
|
25
|
+
- uses: actions/checkout@v4
|
|
29
26
|
- uses: ruby/setup-ruby@v1
|
|
30
27
|
with:
|
|
31
28
|
ruby-version: ${{ matrix.ruby }}
|
|
32
|
-
|
|
33
|
-
run: |
|
|
34
|
-
gem install bundler
|
|
35
|
-
bundle install --jobs 4 --retry 3
|
|
29
|
+
bundler-cache: true
|
|
36
30
|
- name: test
|
|
37
31
|
run: |
|
|
38
32
|
bundle exec parallel_rspec spec/ -n 4 --only-group ${{ matrix.test-group }} --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
|
data/.gitignore
CHANGED
|
@@ -1,11 +1,58 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
*.gem
|
|
2
|
+
*.rbc
|
|
3
|
+
/.config
|
|
4
4
|
/coverage/
|
|
5
|
-
/
|
|
5
|
+
/InstalledFiles
|
|
6
6
|
/pkg/
|
|
7
7
|
/spec/reports/
|
|
8
|
+
/spec/examples.txt
|
|
9
|
+
/test/tmp/
|
|
10
|
+
/test/version_tmp/
|
|
8
11
|
/tmp/
|
|
9
12
|
|
|
10
|
-
#
|
|
11
|
-
.
|
|
13
|
+
# Used by dotenv library to load environment variables.
|
|
14
|
+
# .env
|
|
15
|
+
|
|
16
|
+
# Ignore Byebug command history file.
|
|
17
|
+
.byebug_history
|
|
18
|
+
|
|
19
|
+
## Specific to RubyMotion:
|
|
20
|
+
.dat*
|
|
21
|
+
.repl_history
|
|
22
|
+
build/
|
|
23
|
+
*.bridgesupport
|
|
24
|
+
build-iPhoneOS/
|
|
25
|
+
build-iPhoneSimulator/
|
|
26
|
+
|
|
27
|
+
## Specific to RubyMotion (use of CocoaPods):
|
|
28
|
+
#
|
|
29
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
|
30
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
|
31
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
|
32
|
+
#
|
|
33
|
+
# vendor/Pods/
|
|
34
|
+
|
|
35
|
+
## Documentation cache and generated files:
|
|
36
|
+
/.yardoc/
|
|
37
|
+
/_yardoc/
|
|
38
|
+
/doc/
|
|
39
|
+
/rdoc/
|
|
40
|
+
|
|
41
|
+
## Environment normalization:
|
|
42
|
+
/.bundle/
|
|
43
|
+
/vendor/bundle
|
|
44
|
+
/lib/bundler/man/
|
|
45
|
+
|
|
46
|
+
# for a library or gem, you might want to ignore these files since the code is
|
|
47
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
48
|
+
Gemfile.lock
|
|
49
|
+
.ruby-version
|
|
50
|
+
.ruby-gemset
|
|
51
|
+
|
|
52
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
|
53
|
+
.rvmrc
|
|
54
|
+
|
|
55
|
+
# Used by RuboCop. Remote config files pulled in from inherit_from directive.
|
|
56
|
+
# .rubocop-https?--*
|
|
57
|
+
|
|
58
|
+
.idea
|
data/.rubocop.yml
CHANGED
data/Dockerfile
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
ARG RUBY_VERSION="2.5"
|
|
2
|
+
FROM ruby:$RUBY_VERSION-alpine
|
|
3
|
+
|
|
4
|
+
ARG BUNDLER_VERSION="2.3"
|
|
5
|
+
RUN gem install bundler -v $BUNDLER_VERSION
|
|
6
|
+
|
|
7
|
+
RUN apk add --no-cache --update --upgrade \
|
|
8
|
+
git make gcc libc-dev
|
|
9
|
+
|
|
10
|
+
WORKDIR /app
|
|
11
|
+
|
|
12
|
+
COPY crawler_detect.gemspec ./
|
|
13
|
+
COPY Gemfile ./
|
|
14
|
+
COPY lib/crawler_detect/version.rb ./lib/crawler_detect/
|
|
15
|
+
|
|
16
|
+
RUN bundle install
|
|
17
|
+
|
|
18
|
+
COPY . .
|
|
19
|
+
|
|
20
|
+
CMD ["bundle", "exec", "rubocop"]
|
data/README.md
CHANGED
|
@@ -17,6 +17,10 @@ Comparing with other popular bot-detection gems:
|
|
|
17
17
|
|
|
18
18
|
In order to remain up-to-date, this gem does not accept any crawler data updates – any PRs to edit the crawler data should be offered to the original [JayBizzle/CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect) project.
|
|
19
19
|
|
|
20
|
+
## Requirements
|
|
21
|
+
|
|
22
|
+
- Ruby: MRI 2.5+ or JRuby 9.3+.
|
|
23
|
+
|
|
20
24
|
## Installation
|
|
21
25
|
Add this line to your application's Gemfile:
|
|
22
26
|
|
|
@@ -69,5 +73,14 @@ end
|
|
|
69
73
|
```
|
|
70
74
|
Make sure that your files are correct JSON files.
|
|
71
75
|
Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information.
|
|
76
|
+
|
|
77
|
+
## Development
|
|
78
|
+
|
|
79
|
+
You can run `rubocop` \ `rspec` with any ruby version using docker like this:
|
|
80
|
+
```bash
|
|
81
|
+
docker build --build-arg RUBY_VERSION=3.3 --build-arg BUNDLER_VERSION=2.5 -t crawler_detect:3.3 .
|
|
82
|
+
docker run -it crawler_detect:3.3 bundle exec rspec
|
|
83
|
+
```
|
|
84
|
+
|
|
72
85
|
## License
|
|
73
86
|
MIT License
|
data/crawler_detect.gemspec
CHANGED
|
@@ -24,15 +24,15 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
25
25
|
spec.require_paths = ["lib"]
|
|
26
26
|
|
|
27
|
-
spec.add_dependency "qonfig", "
|
|
27
|
+
spec.add_dependency "qonfig", ">= 0.24"
|
|
28
28
|
|
|
29
|
-
spec.add_development_dependency "activesupport", "
|
|
30
|
-
spec.add_development_dependency "
|
|
31
|
-
spec.add_development_dependency "
|
|
32
|
-
spec.add_development_dependency "
|
|
33
|
-
spec.add_development_dependency "
|
|
34
|
-
spec.add_development_dependency "
|
|
35
|
-
spec.add_development_dependency "
|
|
36
|
-
|
|
37
|
-
spec.
|
|
29
|
+
spec.add_development_dependency "activesupport", ">= 6.0"
|
|
30
|
+
spec.add_development_dependency "fuubar", ">= 2.5"
|
|
31
|
+
spec.add_development_dependency "parallel_tests", ">= 3.0"
|
|
32
|
+
spec.add_development_dependency "rack-test", ">= 2.1"
|
|
33
|
+
spec.add_development_dependency "rake", ">= 13.1"
|
|
34
|
+
spec.add_development_dependency "rspec", ">= 3.13"
|
|
35
|
+
spec.add_development_dependency "armitage-rubocop", ">= 0.82"
|
|
36
|
+
|
|
37
|
+
spec.required_ruby_version = ">= 2.5.0"
|
|
38
38
|
end
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
version: "3"
|
|
2
|
+
services:
|
|
3
|
+
ruby2.5:
|
|
4
|
+
build:
|
|
5
|
+
context: ./
|
|
6
|
+
dockerfile: Dockerfile
|
|
7
|
+
args:
|
|
8
|
+
RUBY_VERSION: 2.5
|
|
9
|
+
BUNDLER_VERSION: 2.3
|
|
10
|
+
ruby2.6:
|
|
11
|
+
build:
|
|
12
|
+
context: ./
|
|
13
|
+
dockerfile: Dockerfile
|
|
14
|
+
args:
|
|
15
|
+
RUBY_VERSION: 2.6
|
|
16
|
+
BUNDLER_VERSION: 2.4
|
|
17
|
+
ruby2.7:
|
|
18
|
+
build:
|
|
19
|
+
context: ./
|
|
20
|
+
dockerfile: Dockerfile
|
|
21
|
+
args:
|
|
22
|
+
RUBY_VERSION: 2.7
|
|
23
|
+
BUNDLER_VERSION: 2.4
|
|
24
|
+
ruby3.0:
|
|
25
|
+
build:
|
|
26
|
+
context: ./
|
|
27
|
+
dockerfile: Dockerfile
|
|
28
|
+
args:
|
|
29
|
+
RUBY_VERSION: 3.0
|
|
30
|
+
BUNDLER_VERSION: 2.4
|
|
31
|
+
ruby3.3:
|
|
32
|
+
build:
|
|
33
|
+
context: ./
|
|
34
|
+
dockerfile: Dockerfile
|
|
35
|
+
args:
|
|
36
|
+
RUBY_VERSION: 3.3
|
|
37
|
+
BUNDLER_VERSION: 2.5
|
|
@@ -11,9 +11,7 @@ module CrawlerDetect
|
|
|
11
11
|
|
|
12
12
|
# @return [true, false] Is User-agent a crawler?
|
|
13
13
|
def is_crawler?
|
|
14
|
-
@is_crawler ||=
|
|
15
|
-
!completely_exclusion? && matches_crawler_list?
|
|
16
|
-
end
|
|
14
|
+
@is_crawler ||= !completely_exclusion? && matches_crawler_list?
|
|
17
15
|
end
|
|
18
16
|
|
|
19
17
|
# @return [String] The detected crawler name from RAW data
|
|
@@ -28,7 +26,7 @@ module CrawlerDetect
|
|
|
28
26
|
# @return [true, false] Is User-agent in white-list?
|
|
29
27
|
def completely_exclusion?
|
|
30
28
|
@user_agent.gsub!(exclusions_matcher, "")
|
|
31
|
-
@user_agent.strip.
|
|
29
|
+
@user_agent.strip.empty?
|
|
32
30
|
end
|
|
33
31
|
|
|
34
32
|
# @private
|
|
@@ -1 +1 @@
|
|
|
1
|
-
[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
|
|
1
|
+
[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^Amazon CloudFront","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookcatalog","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","GeedoProductSearch","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","link-check","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MicrosoftPreview","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","nbertaupete95","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probely\\.com","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","pulsetic\\.com","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]
|
data/lib/rack/crawler_detect.rb
CHANGED
data/rspec-notes
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
1) CrawlerDetect::Detector black list Mozilla/5.0 AppleWebKit/537.36 Chrome/114.0.5735.179 Safari/537.36 Google-Ads-Conversions
|
|
2
|
+
is expected to equal true
|
|
3
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
4
|
+
|
|
5
|
+
expected true
|
|
6
|
+
got false
|
|
7
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
2) CrawlerDetect::Detector black list Amazon CloudFront
|
|
11
|
+
is expected to equal true
|
|
12
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
13
|
+
|
|
14
|
+
expected true
|
|
15
|
+
got false
|
|
16
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
3) CrawlerDetect::Detector black list link-check/5.2.0
|
|
20
|
+
is expected to equal true
|
|
21
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
22
|
+
|
|
23
|
+
expected true
|
|
24
|
+
got false
|
|
25
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
4) CrawlerDetect::Detector black list Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com is expected to equal true
|
|
29
|
+
Failure/Error: it { is_expected.to be(true) }
|
|
30
|
+
|
|
31
|
+
expected true
|
|
32
|
+
got false
|
|
33
|
+
# ./spec/crawler_detect/detector_spec.rb:19:in `block (5 levels) in <top (required)>'
|
metadata
CHANGED
|
@@ -1,153 +1,125 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawler_detect
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.2.
|
|
4
|
+
version: 1.2.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pavel Kozlov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2024-03-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: qonfig
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "
|
|
17
|
+
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
19
|
version: '0.24'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
|
-
- - "
|
|
24
|
+
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0.24'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: activesupport
|
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
|
30
|
-
requirements:
|
|
31
|
-
- - "~>"
|
|
32
|
-
- !ruby/object:Gem::Version
|
|
33
|
-
version: 6.0.3
|
|
34
|
-
type: :development
|
|
35
|
-
prerelease: false
|
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
-
requirements:
|
|
38
|
-
- - "~>"
|
|
39
|
-
- !ruby/object:Gem::Version
|
|
40
|
-
version: 6.0.3
|
|
41
|
-
- !ruby/object:Gem::Dependency
|
|
42
|
-
name: bundler
|
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
|
44
30
|
requirements:
|
|
45
31
|
- - ">="
|
|
46
32
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '
|
|
33
|
+
version: '6.0'
|
|
48
34
|
type: :development
|
|
49
35
|
prerelease: false
|
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
37
|
requirements:
|
|
52
38
|
- - ">="
|
|
53
39
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '
|
|
40
|
+
version: '6.0'
|
|
55
41
|
- !ruby/object:Gem::Dependency
|
|
56
42
|
name: fuubar
|
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
|
58
44
|
requirements:
|
|
59
|
-
- - "
|
|
45
|
+
- - ">="
|
|
60
46
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '2.
|
|
47
|
+
version: '2.5'
|
|
62
48
|
type: :development
|
|
63
49
|
prerelease: false
|
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
51
|
requirements:
|
|
66
|
-
- - "
|
|
52
|
+
- - ">="
|
|
67
53
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '2.
|
|
54
|
+
version: '2.5'
|
|
69
55
|
- !ruby/object:Gem::Dependency
|
|
70
56
|
name: parallel_tests
|
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
|
72
58
|
requirements:
|
|
73
|
-
- - "
|
|
74
|
-
- !ruby/object:Gem::Version
|
|
75
|
-
version: '2.0'
|
|
76
|
-
type: :development
|
|
77
|
-
prerelease: false
|
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
-
requirements:
|
|
80
|
-
- - "~>"
|
|
81
|
-
- !ruby/object:Gem::Version
|
|
82
|
-
version: '2.0'
|
|
83
|
-
- !ruby/object:Gem::Dependency
|
|
84
|
-
name: pry-meta
|
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
|
86
|
-
requirements:
|
|
87
|
-
- - "~>"
|
|
59
|
+
- - ">="
|
|
88
60
|
- !ruby/object:Gem::Version
|
|
89
|
-
version:
|
|
61
|
+
version: '3.0'
|
|
90
62
|
type: :development
|
|
91
63
|
prerelease: false
|
|
92
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
65
|
requirements:
|
|
94
|
-
- - "
|
|
66
|
+
- - ">="
|
|
95
67
|
- !ruby/object:Gem::Version
|
|
96
|
-
version:
|
|
68
|
+
version: '3.0'
|
|
97
69
|
- !ruby/object:Gem::Dependency
|
|
98
70
|
name: rack-test
|
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
|
100
72
|
requirements:
|
|
101
|
-
- - "
|
|
73
|
+
- - ">="
|
|
102
74
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: '
|
|
75
|
+
version: '2.1'
|
|
104
76
|
type: :development
|
|
105
77
|
prerelease: false
|
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
107
79
|
requirements:
|
|
108
|
-
- - "
|
|
80
|
+
- - ">="
|
|
109
81
|
- !ruby/object:Gem::Version
|
|
110
|
-
version: '
|
|
82
|
+
version: '2.1'
|
|
111
83
|
- !ruby/object:Gem::Dependency
|
|
112
84
|
name: rake
|
|
113
85
|
requirement: !ruby/object:Gem::Requirement
|
|
114
86
|
requirements:
|
|
115
87
|
- - ">="
|
|
116
88
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
89
|
+
version: '13.1'
|
|
118
90
|
type: :development
|
|
119
91
|
prerelease: false
|
|
120
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
93
|
requirements:
|
|
122
94
|
- - ">="
|
|
123
95
|
- !ruby/object:Gem::Version
|
|
124
|
-
version: '
|
|
96
|
+
version: '13.1'
|
|
125
97
|
- !ruby/object:Gem::Dependency
|
|
126
98
|
name: rspec
|
|
127
99
|
requirement: !ruby/object:Gem::Requirement
|
|
128
100
|
requirements:
|
|
129
|
-
- - "
|
|
101
|
+
- - ">="
|
|
130
102
|
- !ruby/object:Gem::Version
|
|
131
|
-
version: '3.
|
|
103
|
+
version: '3.13'
|
|
132
104
|
type: :development
|
|
133
105
|
prerelease: false
|
|
134
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
135
107
|
requirements:
|
|
136
|
-
- - "
|
|
108
|
+
- - ">="
|
|
137
109
|
- !ruby/object:Gem::Version
|
|
138
|
-
version: '3.
|
|
110
|
+
version: '3.13'
|
|
139
111
|
- !ruby/object:Gem::Dependency
|
|
140
112
|
name: armitage-rubocop
|
|
141
113
|
requirement: !ruby/object:Gem::Requirement
|
|
142
114
|
requirements:
|
|
143
|
-
- -
|
|
115
|
+
- - ">="
|
|
144
116
|
- !ruby/object:Gem::Version
|
|
145
117
|
version: '0.82'
|
|
146
118
|
type: :development
|
|
147
119
|
prerelease: false
|
|
148
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
149
121
|
requirements:
|
|
150
|
-
- -
|
|
122
|
+
- - ">="
|
|
151
123
|
- !ruby/object:Gem::Version
|
|
152
124
|
version: '0.82'
|
|
153
125
|
description: CrawlerDetect is a library to detect bots/crawlers via the user agent
|
|
@@ -157,18 +129,20 @@ executables: []
|
|
|
157
129
|
extensions: []
|
|
158
130
|
extra_rdoc_files: []
|
|
159
131
|
files:
|
|
132
|
+
- ".dockerignore"
|
|
160
133
|
- ".github/workflows/build.yml"
|
|
161
134
|
- ".gitignore"
|
|
162
135
|
- ".rspec"
|
|
163
136
|
- ".rubocop.yml"
|
|
164
137
|
- CHANGELOG.md
|
|
138
|
+
- Dockerfile
|
|
165
139
|
- Gemfile
|
|
166
|
-
- Gemfile.lock
|
|
167
140
|
- LICENSE.txt
|
|
168
141
|
- README.md
|
|
169
142
|
- Rakefile
|
|
170
143
|
- bin/update_raw_files
|
|
171
144
|
- crawler_detect.gemspec
|
|
145
|
+
- docker-compose.yml
|
|
172
146
|
- lib/crawler_detect.rb
|
|
173
147
|
- lib/crawler_detect/config.rb
|
|
174
148
|
- lib/crawler_detect/detector.rb
|
|
@@ -182,6 +156,7 @@ files:
|
|
|
182
156
|
- lib/crawler_detect/library/raw/Headers.json
|
|
183
157
|
- lib/crawler_detect/version.rb
|
|
184
158
|
- lib/rack/crawler_detect.rb
|
|
159
|
+
- rspec-notes
|
|
185
160
|
homepage: https://github.com/loadkpi/crawler_detect
|
|
186
161
|
licenses:
|
|
187
162
|
- MIT
|
|
@@ -194,14 +169,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
194
169
|
requirements:
|
|
195
170
|
- - ">="
|
|
196
171
|
- !ruby/object:Gem::Version
|
|
197
|
-
version:
|
|
172
|
+
version: 2.5.0
|
|
198
173
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
174
|
requirements:
|
|
200
175
|
- - ">="
|
|
201
176
|
- !ruby/object:Gem::Version
|
|
202
177
|
version: '0'
|
|
203
178
|
requirements: []
|
|
204
|
-
rubygems_version: 3.
|
|
179
|
+
rubygems_version: 3.2.33
|
|
205
180
|
signing_key:
|
|
206
181
|
specification_version: 4
|
|
207
182
|
summary: 'CrawlerDetect: detect bots/crawlers'
|
data/Gemfile.lock
DELETED
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
crawler_detect (1.2.3)
|
|
5
|
-
qonfig (~> 0.24)
|
|
6
|
-
|
|
7
|
-
GEM
|
|
8
|
-
remote: https://rubygems.org/
|
|
9
|
-
specs:
|
|
10
|
-
activesupport (6.0.6)
|
|
11
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
12
|
-
i18n (>= 0.7, < 2)
|
|
13
|
-
minitest (~> 5.1)
|
|
14
|
-
tzinfo (~> 1.1)
|
|
15
|
-
zeitwerk (~> 2.2, >= 2.2.2)
|
|
16
|
-
armitage-rubocop (0.82.0)
|
|
17
|
-
rubocop (= 0.82.0)
|
|
18
|
-
rubocop-performance (= 1.5.2)
|
|
19
|
-
rubocop-rails (= 2.5.2)
|
|
20
|
-
rubocop-rake (= 0.5.1)
|
|
21
|
-
rubocop-rspec (= 1.38.1)
|
|
22
|
-
ast (2.4.2)
|
|
23
|
-
awesome_print (1.9.2)
|
|
24
|
-
byebug (11.1.3)
|
|
25
|
-
coderay (1.1.3)
|
|
26
|
-
concurrent-ruby (1.1.10)
|
|
27
|
-
diff-lcs (1.5.0)
|
|
28
|
-
fuubar (2.5.1)
|
|
29
|
-
rspec-core (~> 3.0)
|
|
30
|
-
ruby-progressbar (~> 1.4)
|
|
31
|
-
i18n (1.12.0)
|
|
32
|
-
concurrent-ruby (~> 1.0)
|
|
33
|
-
jaro_winkler (1.5.4)
|
|
34
|
-
method_source (1.0.0)
|
|
35
|
-
minitest (5.16.3)
|
|
36
|
-
parallel (1.22.1)
|
|
37
|
-
parallel_tests (2.32.0)
|
|
38
|
-
parallel
|
|
39
|
-
parser (3.1.3.0)
|
|
40
|
-
ast (~> 2.4.1)
|
|
41
|
-
pry (0.14.1)
|
|
42
|
-
coderay (~> 1.1)
|
|
43
|
-
method_source (~> 1.0)
|
|
44
|
-
pry-byebug (3.10.1)
|
|
45
|
-
byebug (~> 11.0)
|
|
46
|
-
pry (>= 0.13, < 0.15)
|
|
47
|
-
pry-meta (0.0.10)
|
|
48
|
-
awesome_print
|
|
49
|
-
pry
|
|
50
|
-
pry-byebug
|
|
51
|
-
pry-remote
|
|
52
|
-
pry-remote (0.1.8)
|
|
53
|
-
pry (~> 0.9)
|
|
54
|
-
slop (~> 3.0)
|
|
55
|
-
qonfig (0.28.0)
|
|
56
|
-
rack (2.2.4)
|
|
57
|
-
rack-test (1.1.0)
|
|
58
|
-
rack (>= 1.0, < 3)
|
|
59
|
-
rainbow (3.1.1)
|
|
60
|
-
rake (13.0.6)
|
|
61
|
-
rexml (3.2.5)
|
|
62
|
-
rspec (3.12.0)
|
|
63
|
-
rspec-core (~> 3.12.0)
|
|
64
|
-
rspec-expectations (~> 3.12.0)
|
|
65
|
-
rspec-mocks (~> 3.12.0)
|
|
66
|
-
rspec-core (3.12.0)
|
|
67
|
-
rspec-support (~> 3.12.0)
|
|
68
|
-
rspec-expectations (3.12.0)
|
|
69
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
70
|
-
rspec-support (~> 3.12.0)
|
|
71
|
-
rspec-mocks (3.12.0)
|
|
72
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
73
|
-
rspec-support (~> 3.12.0)
|
|
74
|
-
rspec-support (3.12.0)
|
|
75
|
-
rubocop (0.82.0)
|
|
76
|
-
jaro_winkler (~> 1.5.1)
|
|
77
|
-
parallel (~> 1.10)
|
|
78
|
-
parser (>= 2.7.0.1)
|
|
79
|
-
rainbow (>= 2.2.2, < 4.0)
|
|
80
|
-
rexml
|
|
81
|
-
ruby-progressbar (~> 1.7)
|
|
82
|
-
unicode-display_width (>= 1.4.0, < 2.0)
|
|
83
|
-
rubocop-performance (1.5.2)
|
|
84
|
-
rubocop (>= 0.71.0)
|
|
85
|
-
rubocop-rails (2.5.2)
|
|
86
|
-
activesupport
|
|
87
|
-
rack (>= 1.1)
|
|
88
|
-
rubocop (>= 0.72.0)
|
|
89
|
-
rubocop-rake (0.5.1)
|
|
90
|
-
rubocop
|
|
91
|
-
rubocop-rspec (1.38.1)
|
|
92
|
-
rubocop (>= 0.68.1)
|
|
93
|
-
ruby-progressbar (1.11.0)
|
|
94
|
-
slop (3.6.0)
|
|
95
|
-
thread_safe (0.3.6)
|
|
96
|
-
tzinfo (1.2.10)
|
|
97
|
-
thread_safe (~> 0.1)
|
|
98
|
-
unicode-display_width (1.8.0)
|
|
99
|
-
zeitwerk (2.6.6)
|
|
100
|
-
|
|
101
|
-
PLATFORMS
|
|
102
|
-
ruby
|
|
103
|
-
|
|
104
|
-
DEPENDENCIES
|
|
105
|
-
activesupport (~> 6.0.3)
|
|
106
|
-
armitage-rubocop (= 0.82)
|
|
107
|
-
bundler (>= 1.15)
|
|
108
|
-
crawler_detect!
|
|
109
|
-
fuubar (~> 2.0)
|
|
110
|
-
parallel_tests (~> 2.0)
|
|
111
|
-
pry-meta (~> 0.0.10)
|
|
112
|
-
rack-test (~> 1.1)
|
|
113
|
-
rake (>= 10.0)
|
|
114
|
-
rspec (~> 3.0)
|
|
115
|
-
|
|
116
|
-
BUNDLED WITH
|
|
117
|
-
2.3.21
|