crawler_detect 1.2.11 → 1.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/crawler_detect/version.rb +1 -1
- metadata +16 -30
- data/.dockerignore +0 -1
- data/.github/workflows/build.yml +0 -51
- data/.github/workflows/update_raw_files.yml +0 -50
- data/.gitignore +0 -58
- data/.rspec +0 -3
- data/.rspec_status +0 -169339
- data/.rubocop.yml +0 -20
- data/CHANGELOG.md +0 -40
- data/Dockerfile +0 -20
- data/Gemfile +0 -7
- data/Rakefile +0 -8
- data/bin/update_raw_files +0 -10
- data/crawler_detect.gemspec +0 -44
- data/docker-compose.yml +0 -37
data/.rubocop.yml
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
inherit_gem:
|
|
2
|
-
armitage-rubocop:
|
|
3
|
-
- lib/rubocop.general.yml
|
|
4
|
-
- lib/rubocop.rake.yml
|
|
5
|
-
- lib/rubocop.rspec.yml
|
|
6
|
-
|
|
7
|
-
AllCops:
|
|
8
|
-
TargetRubyVersion: 2.5
|
|
9
|
-
NewCops: enable
|
|
10
|
-
Include:
|
|
11
|
-
- lib/**/*.rb
|
|
12
|
-
- spec/**/*.rb
|
|
13
|
-
- Gemfile
|
|
14
|
-
- Rakefile
|
|
15
|
-
- crawler_detect.gemspec
|
|
16
|
-
- bin/console
|
|
17
|
-
|
|
18
|
-
Style/StringLiterals:
|
|
19
|
-
Enabled: true
|
|
20
|
-
EnforcedStyle: double_quotes
|
data/CHANGELOG.md
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
# CrawlerDetect major changes
|
|
2
|
-
|
|
3
|
-
This changelog **does not contain** raw data updates
|
|
4
|
-
but only major changes.
|
|
5
|
-
|
|
6
|
-
1.2.0
|
|
7
|
-
---------
|
|
8
|
-
- Speed up crawler detection by caching Regexp objects [#38]
|
|
9
|
-
|
|
10
|
-
1.1.0
|
|
11
|
-
---------
|
|
12
|
-
- Moves to Ruby's Stdlib JSON implementation to reduce dependencies
|
|
13
|
-
|
|
14
|
-
1.0.0
|
|
15
|
-
---------
|
|
16
|
-
- Use raw JSON files instead of copy them to rb [#8]
|
|
17
|
-
- Add CrawlerDetect::Config to make it possible to have own raw files [#8]
|
|
18
|
-
- Add bin/update_raw_files to update raw files from PHP lib [#8]
|
|
19
|
-
- Add Changelog
|
|
20
|
-
|
|
21
|
-
0.1.11
|
|
22
|
-
---------
|
|
23
|
-
- Add thread safety [#19]
|
|
24
|
-
|
|
25
|
-
0.1.6
|
|
26
|
-
---------
|
|
27
|
-
- Strip crawler name [#10]
|
|
28
|
-
|
|
29
|
-
0.1.2
|
|
30
|
-
---------
|
|
31
|
-
- Add parallel tests [#2]
|
|
32
|
-
|
|
33
|
-
0.1.1
|
|
34
|
-
---------
|
|
35
|
-
- Fix: rack request
|
|
36
|
-
|
|
37
|
-
0.1.0
|
|
38
|
-
---------
|
|
39
|
-
- init
|
|
40
|
-
|
data/Dockerfile
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
ARG RUBY_VERSION="2.5"
|
|
2
|
-
FROM ruby:$RUBY_VERSION-alpine
|
|
3
|
-
|
|
4
|
-
ARG BUNDLER_VERSION="2.3"
|
|
5
|
-
RUN gem install bundler -v $BUNDLER_VERSION
|
|
6
|
-
|
|
7
|
-
RUN apk add --no-cache --update --upgrade \
|
|
8
|
-
git make gcc libc-dev
|
|
9
|
-
|
|
10
|
-
WORKDIR /app
|
|
11
|
-
|
|
12
|
-
COPY crawler_detect.gemspec ./
|
|
13
|
-
COPY Gemfile ./
|
|
14
|
-
COPY lib/crawler_detect/version.rb ./lib/crawler_detect/
|
|
15
|
-
|
|
16
|
-
RUN bundle install
|
|
17
|
-
|
|
18
|
-
COPY . .
|
|
19
|
-
|
|
20
|
-
CMD ["bundle", "exec", "rubocop"]
|
data/Gemfile
DELETED
data/Rakefile
DELETED
data/bin/update_raw_files
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
|
|
3
|
-
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
|
4
|
-
|
|
5
|
-
wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
|
|
6
|
-
wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
|
|
7
|
-
wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
|
|
8
|
-
|
|
9
|
-
wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/crawlers.txt
|
|
10
|
-
wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/devices.txt
|
data/crawler_detect.gemspec
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
lib = File.expand_path("lib", __dir__)
|
|
4
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
-
require "crawler_detect/version"
|
|
6
|
-
|
|
7
|
-
Gem::Specification.new do |spec|
|
|
8
|
-
spec.name = "crawler_detect"
|
|
9
|
-
spec.version = CrawlerDetect::VERSION
|
|
10
|
-
spec.authors = ["Pavel Kozlov"]
|
|
11
|
-
spec.email = ["loadkpi@gmail.com"]
|
|
12
|
-
|
|
13
|
-
spec.summary = "CrawlerDetect: detect bots/crawlers"
|
|
14
|
-
spec.description = "CrawlerDetect is a library to detect bots/crawlers via the user agent"
|
|
15
|
-
spec.homepage = "https://github.com/loadkpi/crawler_detect"
|
|
16
|
-
spec.license = "MIT"
|
|
17
|
-
|
|
18
|
-
# Specify which files should be added to the gem when it is released.
|
|
19
|
-
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
20
|
-
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
21
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
22
|
-
end
|
|
23
|
-
spec.bindir = "exe"
|
|
24
|
-
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
25
|
-
spec.require_paths = ["lib"]
|
|
26
|
-
|
|
27
|
-
spec.add_dependency "qonfig", ">= 0.24"
|
|
28
|
-
|
|
29
|
-
spec.add_development_dependency "activesupport", ">= 6.0"
|
|
30
|
-
spec.add_development_dependency "fuubar", ">= 2.5"
|
|
31
|
-
spec.add_development_dependency "parallel_tests", ">= 3.0"
|
|
32
|
-
spec.add_development_dependency "rack-test", ">= 2.1"
|
|
33
|
-
spec.add_development_dependency "rake", ">= 13.1"
|
|
34
|
-
spec.add_development_dependency "rspec", ">= 3.13"
|
|
35
|
-
spec.add_development_dependency "armitage-rubocop", ">= 0.82"
|
|
36
|
-
|
|
37
|
-
spec.required_ruby_version = ">= 2.5.0"
|
|
38
|
-
|
|
39
|
-
spec.metadata = {
|
|
40
|
-
"bug_tracker_uri" => "https://github.com/loadkpi/crawler_detect/issues",
|
|
41
|
-
"source_code_uri" => "https://github.com/loadkpi/crawler_detect",
|
|
42
|
-
"changelog_uri" => "https://github.com/loadkpi/crawler_detect/blob/master/CHANGELOG.md"
|
|
43
|
-
}
|
|
44
|
-
end
|
data/docker-compose.yml
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
version: "3"
|
|
2
|
-
services:
|
|
3
|
-
ruby2.5:
|
|
4
|
-
build:
|
|
5
|
-
context: ./
|
|
6
|
-
dockerfile: Dockerfile
|
|
7
|
-
args:
|
|
8
|
-
RUBY_VERSION: 2.5
|
|
9
|
-
BUNDLER_VERSION: 2.3
|
|
10
|
-
ruby2.6:
|
|
11
|
-
build:
|
|
12
|
-
context: ./
|
|
13
|
-
dockerfile: Dockerfile
|
|
14
|
-
args:
|
|
15
|
-
RUBY_VERSION: 2.6
|
|
16
|
-
BUNDLER_VERSION: 2.4
|
|
17
|
-
ruby2.7:
|
|
18
|
-
build:
|
|
19
|
-
context: ./
|
|
20
|
-
dockerfile: Dockerfile
|
|
21
|
-
args:
|
|
22
|
-
RUBY_VERSION: 2.7
|
|
23
|
-
BUNDLER_VERSION: 2.4
|
|
24
|
-
ruby3.0:
|
|
25
|
-
build:
|
|
26
|
-
context: ./
|
|
27
|
-
dockerfile: Dockerfile
|
|
28
|
-
args:
|
|
29
|
-
RUBY_VERSION: 3.0
|
|
30
|
-
BUNDLER_VERSION: 2.4
|
|
31
|
-
ruby3.3:
|
|
32
|
-
build:
|
|
33
|
-
context: ./
|
|
34
|
-
dockerfile: Dockerfile
|
|
35
|
-
args:
|
|
36
|
-
RUBY_VERSION: 3.3
|
|
37
|
-
BUNDLER_VERSION: 2.5
|