sitemap_check 0.1.9 → 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.jrubyrc +1 -0
- data/.travis.yml +2 -1
- data/CODE_OF_CONDUCT.md +72 -2
- data/Dockerfile +1 -1
- data/Gemfile +1 -0
- data/README.md +9 -0
- data/Rakefile +4 -3
- data/bin/sitemap_check +1 -0
- data/lib/sitemap_check.rb +1 -0
- data/lib/sitemap_check/logger.rb +1 -0
- data/lib/sitemap_check/page.rb +24 -2
- data/lib/sitemap_check/sitemap.rb +2 -1
- data/lib/sitemap_check/validator.rb +61 -0
- data/lib/sitemap_check/version.rb +2 -1
- data/sitemap_check.gemspec +2 -0
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 301e1ef8730ab4b8171e6720d17642e84a940ea2
|
4
|
+
data.tar.gz: bd043c3da703584c5cff25e8853d63b8c93f0035
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 193b088cff5eaefbc56806c7204428fb3426821bea58b76437fdd57da59e950da5d07a2939aff472f26f7402cf4448c7c3e4ad8a1f61e9da5202c193ced47cbb
|
7
|
+
data.tar.gz: d108143a6fd8552d52ec9c8a8c6dbcbfb55b11afd6036685742c60ed8f6d55d3e00443541c5651f01ba29ad901bf97e665e1456d443f8b5f0e66485a2f8ad7eb
|
data/.jrubyrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
debug.fullTrace=true
|
data/.travis.yml
CHANGED
data/CODE_OF_CONDUCT.md
CHANGED
@@ -1,3 +1,73 @@
|
|
1
|
-
# Contributor Code of Conduct
|
1
|
+
# Contributor Covenant Code of Conduct
|
2
2
|
|
3
|
-
|
3
|
+
## Our Pledge
|
4
|
+
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
10
|
+
orientation.
|
11
|
+
|
12
|
+
## Our Standards
|
13
|
+
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
15
|
+
include:
|
16
|
+
|
17
|
+
* Using welcoming and inclusive language
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
19
|
+
* Gracefully accepting constructive criticism
|
20
|
+
* Focusing on what is best for the community
|
21
|
+
* Showing empathy towards other community members
|
22
|
+
|
23
|
+
Examples of unacceptable behavior by participants include:
|
24
|
+
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
26
|
+
advances
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
28
|
+
* Public or private harassment
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
30
|
+
address, without explicit permission
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
32
|
+
professional setting
|
33
|
+
|
34
|
+
## Our Responsibilities
|
35
|
+
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
38
|
+
response to any instances of unacceptable behavior.
|
39
|
+
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
44
|
+
threatening, offensive, or harmful.
|
45
|
+
|
46
|
+
## Scope
|
47
|
+
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
49
|
+
when an individual is representing the project or its community. Examples of
|
50
|
+
representing a project or community include using an official project e-mail
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
53
|
+
further defined and clarified by project maintainers.
|
54
|
+
|
55
|
+
## Enforcement
|
56
|
+
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
58
|
+
reported by contacting the project team at devops@reevoo.com. All
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
63
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
64
|
+
faith may face temporary or permanent repercussions as determined by other
|
65
|
+
members of the project's leadership.
|
66
|
+
|
67
|
+
## Attribution
|
68
|
+
|
69
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
70
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
71
|
+
|
72
|
+
[homepage]: http://contributor-covenant.org
|
73
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
data/Dockerfile
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# Sitemap Check
|
2
2
|
|
3
|
+
[![Docker Repository on Quay](https://quay.io/repository/reevoo/sitemap_check/status "Docker Repository on Quay")](https://quay.io/repository/reevoo/sitemap_check)
|
3
4
|
[![Build Status](https://travis-ci.org/reevoo/sitemap_check.svg?branch=master)](https://travis-ci.org/reevoo/sitemap_check)
|
4
5
|
[![Code Climate](https://codeclimate.com/github/reevoo/sitemap_check/badges/gpa.svg)](https://codeclimate.com/github/reevoo/sitemap_check)
|
5
6
|
[![Test Coverage](https://codeclimate.com/github/reevoo/sitemap_check/badges/coverage.svg)](https://codeclimate.com/github/reevoo/sitemap_check/coverage)
|
@@ -21,6 +22,14 @@ $ CHECK_URL=http://www.reevoo.com/sitemap_index.xml sitemap_check
|
|
21
22
|
$ sitemap_check http://www.reevoo.com/sitemap_index.xml
|
22
23
|
```
|
23
24
|
|
25
|
+
You can also run `sitemap_check` in validation mode:
|
26
|
+
|
27
|
+
```bash
|
28
|
+
$ VALIDATE=1 sitemap_check http://www.reevoo.com/sitemap_index.xml
|
29
|
+
```
|
30
|
+
|
31
|
+
This will validate response bodies with W3C's validation service.
|
32
|
+
|
24
33
|
# Docker
|
25
34
|
|
26
35
|
```bash
|
data/Rakefile
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require "bundler/gem_tasks"
|
2
3
|
require "reevoocop/rake_task"
|
3
4
|
require "rspec/core/rake_task"
|
@@ -7,9 +8,9 @@ RSpec::Core::RakeTask.new(:spec)
|
|
7
8
|
|
8
9
|
DOCKER_REPO = "quay.io/reevoo/sitemap_check"
|
9
10
|
|
10
|
-
task default: [
|
11
|
-
task release: [
|
12
|
-
task build: [
|
11
|
+
task default: %i[spec reevoocop]
|
12
|
+
task release: %i[spec reevoocop]
|
13
|
+
task build: %i[spec reevoocop]
|
13
14
|
|
14
15
|
task :release do
|
15
16
|
sh "docker build --build-arg VERSION=#{SitemapCheck::VERSION} -t #{DOCKER_REPO}:#{SitemapCheck::VERSION} ."
|
data/bin/sitemap_check
CHANGED
data/lib/sitemap_check.rb
CHANGED
data/lib/sitemap_check/logger.rb
CHANGED
data/lib/sitemap_check/page.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require "typhoeus"
|
2
3
|
require "sitemap_check/logger"
|
4
|
+
require "sitemap_check/validator"
|
3
5
|
require "colorize"
|
4
6
|
require "uri"
|
5
7
|
|
@@ -7,9 +9,16 @@ class SitemapCheck
|
|
7
9
|
class Page
|
8
10
|
def initialize(url, logger = Logger.new)
|
9
11
|
self.uri = URI(url)
|
10
|
-
replace_host
|
11
|
-
self.request = Typhoeus::Request.new(self.url, method: :head, followlocation: true)
|
12
12
|
self.logger = logger
|
13
|
+
|
14
|
+
replace_host
|
15
|
+
|
16
|
+
self.request = Typhoeus::Request.new(
|
17
|
+
self.url,
|
18
|
+
method: request_method,
|
19
|
+
followlocation: true,
|
20
|
+
)
|
21
|
+
|
13
22
|
setup_callbacks
|
14
23
|
end
|
15
24
|
|
@@ -32,6 +41,7 @@ class SitemapCheck
|
|
32
41
|
def setup_callbacks # rubocop:disable Metrics/AbcSize
|
33
42
|
request.on_complete do |response|
|
34
43
|
if response.success?
|
44
|
+
validate(response)
|
35
45
|
@exists = true
|
36
46
|
elsif response.timed_out?
|
37
47
|
@exists = true
|
@@ -45,5 +55,17 @@ class SitemapCheck
|
|
45
55
|
end
|
46
56
|
end
|
47
57
|
end
|
58
|
+
|
59
|
+
def request_method
|
60
|
+
validate? ? :get : :head
|
61
|
+
end
|
62
|
+
|
63
|
+
def validate(response)
|
64
|
+
Validator.new(response, logger).validate if validate?
|
65
|
+
end
|
66
|
+
|
67
|
+
def validate?
|
68
|
+
ENV["VALIDATE"]
|
69
|
+
end
|
48
70
|
end
|
49
71
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require "typhoeus"
|
2
3
|
require "sitemap_check/page"
|
3
4
|
require "sitemap_check/logger"
|
@@ -37,7 +38,7 @@ class SitemapCheck
|
|
37
38
|
pages.select(&:error)
|
38
39
|
end
|
39
40
|
|
40
|
-
def exists?
|
41
|
+
def exists?
|
41
42
|
@ok
|
42
43
|
end
|
43
44
|
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require "w3c_validators"
|
3
|
+
|
4
|
+
class SitemapCheck
|
5
|
+
class Validator
|
6
|
+
LIMIT = 100
|
7
|
+
|
8
|
+
attr_accessor :logger, :response
|
9
|
+
|
10
|
+
class << self
|
11
|
+
attr_accessor :message_count
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(response, logger = Logger.new)
|
15
|
+
self.logger = logger
|
16
|
+
self.response = response
|
17
|
+
self.class.message_count ||= 0
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate
|
21
|
+
validator = W3CValidators::NuValidator.new
|
22
|
+
result = validator.validate_text(response.body)
|
23
|
+
return if result.errors.empty? && result.warnings.empty?
|
24
|
+
|
25
|
+
log_url
|
26
|
+
log_errors(result)
|
27
|
+
log_warnings(result)
|
28
|
+
fail_if_too_many_messages
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def log_url
|
34
|
+
logger.log "-" * 80
|
35
|
+
logger.log response.effective_url.cyan
|
36
|
+
end
|
37
|
+
|
38
|
+
def log_errors(result)
|
39
|
+
result.errors.each do |e|
|
40
|
+
logger.log " ERROR: #{e.message}".red
|
41
|
+
logger.log " #{e.source.inspect}"
|
42
|
+
|
43
|
+
self.class.message_count += 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def log_warnings(result)
|
48
|
+
result.warnings.each do |w|
|
49
|
+
logger.log " WARNING: #{w.message}".yellow
|
50
|
+
logger.log " #{w.source.inspect}"
|
51
|
+
|
52
|
+
self.class.message_count += 1
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def fail_if_too_many_messages
|
57
|
+
error = "Stopping because there are more than #{LIMIT} messages."
|
58
|
+
fail error if self.class.message_count > LIMIT
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/sitemap_check.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
2
3
|
lib = File.expand_path("../lib", __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require "sitemap_check/version"
|
@@ -21,6 +22,7 @@ Gem::Specification.new do |spec|
|
|
21
22
|
spec.add_dependency "nokogiri", "~> 1.7"
|
22
23
|
spec.add_dependency "typhoeus", "~> 1.1"
|
23
24
|
spec.add_dependency "colorize", "~> 0.8"
|
25
|
+
spec.add_dependency "w3c_validators", "~> 1.3"
|
24
26
|
spec.add_development_dependency "bundler", "~> 1.14"
|
25
27
|
spec.add_development_dependency "rake", "~> 12.0"
|
26
28
|
spec.add_development_dependency "rspec", "~> 3.5"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sitemap_check
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ed Robinson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-07-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.8'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: w3c_validators
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.3'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.3'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: bundler
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -145,6 +159,7 @@ extensions: []
|
|
145
159
|
extra_rdoc_files: []
|
146
160
|
files:
|
147
161
|
- ".gitignore"
|
162
|
+
- ".jrubyrc"
|
148
163
|
- ".rspec"
|
149
164
|
- ".ruby-version"
|
150
165
|
- ".travis.yml"
|
@@ -159,6 +174,7 @@ files:
|
|
159
174
|
- lib/sitemap_check/logger.rb
|
160
175
|
- lib/sitemap_check/page.rb
|
161
176
|
- lib/sitemap_check/sitemap.rb
|
177
|
+
- lib/sitemap_check/validator.rb
|
162
178
|
- lib/sitemap_check/version.rb
|
163
179
|
- sitemap_check.gemspec
|
164
180
|
homepage: https://github.com/reevoo/sitemap_check
|