middleman-robots 1.3.4 → 1.3.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +2 -0
- data/.rubocop.yml +5 -16
- data/.travis.yml +5 -8
- data/Gemfile +2 -0
- data/Rakefile +3 -3
- data/features/support/env.rb +3 -1
- data/fixtures/server-app/config.rb +18 -4
- data/lib/middleman-robots.rb +2 -0
- data/lib/middleman-robots/extension.rb +2 -0
- data/lib/middleman-robots/generator.rb +12 -27
- data/lib/middleman-robots/generators/block.rb +55 -0
- data/lib/middleman-robots/generators/blocks.rb +26 -0
- data/lib/middleman-robots/generators/sitemap_uri.rb +24 -0
- data/lib/middleman-robots/version.rb +3 -1
- data/lib/middleman_extension.rb +2 -0
- data/middleman-robots.gemspec +4 -1
- data/spec/lib/middleman-robots/generator_spec.rb +65 -0
- data/spec/lib/middleman-robots/generators/block_spec.rb +205 -0
- data/spec/lib/middleman-robots/generators/blocks_spec.rb +50 -0
- data/spec/lib/middleman-robots/generators/sitemap_uri_spec.rb +51 -0
- data/spec/spec_helper.rb +13 -0
- metadata +32 -8
- data/lib/middleman-robots/group.rb +0 -51
- data/tests/test_generator.rb +0 -43
- data/tests/test_group.rb +0 -120
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bb2ad78f094b46f2d5c0a19e03471eff2285e443039d4872532d44a91764d557
|
4
|
+
data.tar.gz: 3b12a61174e1bebdc5f398c67fb1bfa102bdeb2fc0b7eae7b25378c1d4be98fb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b333b535edd07872e4711106d6c40c0013844078f447500f6ae63a14364010ca746a9d2e37b4a8bee17d8cd28c0d2f5fe7945630765f07abc97ae5cf92fb98ea
|
7
|
+
data.tar.gz: 3aecf7a784a98210286eddb24be8ca8cac9892a5476cad6b92059c00d15953845cfd5c8d3fbbeaca5f7ff392b91ebd9da0f6b6a71140e63890617b6f26c0d1be
|
data/.rspec
ADDED
data/.rubocop.yml
CHANGED
@@ -3,35 +3,24 @@ AllCops:
|
|
3
3
|
- 'pkg/*'
|
4
4
|
- 'tmp/**/*'
|
5
5
|
- 'vendor/**/*'
|
6
|
-
TargetRubyVersion: 2.
|
6
|
+
TargetRubyVersion: 2.4
|
7
7
|
DisplayCopNames: true
|
8
8
|
|
9
|
-
Rails:
|
10
|
-
Enabled: false
|
11
|
-
|
12
9
|
Style/ClassAndModuleChildren:
|
13
10
|
Enabled: false
|
14
11
|
|
15
12
|
Style/Documentation:
|
16
13
|
Enabled: false
|
17
14
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
Style/RedundantSelf:
|
22
|
-
Enabled: false
|
23
|
-
|
24
|
-
Gemspec/RequiredRubyVersion:
|
25
|
-
Enabled: false
|
15
|
+
Metrics/BlockLength:
|
16
|
+
Exclude:
|
17
|
+
- 'spec/**/*_spec.rb'
|
26
18
|
|
27
19
|
Metrics/LineLength:
|
28
|
-
Max:
|
20
|
+
Max: 100
|
29
21
|
Exclude:
|
30
22
|
- '*.gemspec'
|
31
23
|
|
32
|
-
Metrics/MethodLength:
|
33
|
-
Max: 20
|
34
|
-
|
35
24
|
Naming/FileName:
|
36
25
|
Enabled: true
|
37
26
|
Exclude:
|
data/.travis.yml
CHANGED
@@ -5,20 +5,17 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.
|
9
|
-
- 2.
|
10
|
-
- 2.
|
8
|
+
- 2.6.4
|
9
|
+
- 2.5.6
|
10
|
+
- 2.4.7
|
11
11
|
os:
|
12
|
-
- linux
|
12
|
+
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
16
|
- rvm: ruby-head
|
17
17
|
script:
|
18
|
-
- bundle exec rake test
|
19
18
|
- bundle exec rubocop
|
19
|
+
- bundle exec rake test
|
20
20
|
cache: bundler
|
21
21
|
env: TEST=true
|
22
|
-
notifications:
|
23
|
-
slack:
|
24
|
-
secure: C8xb94J4fstzAI7t9GcvKQieKc1k+ECA96ydcxDwBI1fAbupQjz4jafMwnOnD1g9QMe8hlykSNUlCXNCbcA9DZG7COjPZHCY3Icv1+0Dp44bBTDQldR8DPACPMU5Qe+pWoTZViPgJQ7tq2V9TXrGdOnOtkPTwZ9KtZrmNhxW7tM=
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'bundler/gem_tasks'
|
2
4
|
|
3
5
|
task default: :test
|
4
6
|
|
5
7
|
desc 'test command'
|
6
8
|
task :test do
|
7
|
-
|
8
|
-
sh "bundle exec ruby #{f}"
|
9
|
-
end
|
9
|
+
sh 'bundle exec rspec spec/'
|
10
10
|
sh 'bundle exec cucumber features/'
|
11
11
|
end
|
data/features/support/env.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
PROJECT_ROOT_PATH = File.expand_path('../../', __dir__)
|
2
4
|
require 'middleman-core'
|
3
5
|
require 'middleman-core/step_definitions'
|
4
6
|
require File.join(PROJECT_ROOT_PATH, 'lib', 'middleman-robots')
|
@@ -1,13 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
rules = [
|
2
4
|
{
|
3
5
|
user_agent: 'Googlebot',
|
4
|
-
disallow: %w[
|
5
|
-
|
6
|
+
disallow: %w[
|
7
|
+
tmp/*
|
8
|
+
/something/dir/file_disallow.html
|
9
|
+
],
|
10
|
+
allow: %w[
|
11
|
+
allow/*
|
12
|
+
/something/dir/file_allow.html
|
13
|
+
]
|
6
14
|
},
|
7
15
|
{
|
8
16
|
user_agent: 'Googlebot-Image',
|
9
|
-
disallow:
|
10
|
-
|
17
|
+
disallow: %w[
|
18
|
+
tmp/*
|
19
|
+
/something/dir/file_disallow.html
|
20
|
+
],
|
21
|
+
allow: %w[
|
22
|
+
allow/*
|
23
|
+
/something/dir/file_allow.html
|
24
|
+
]
|
11
25
|
}
|
12
26
|
]
|
13
27
|
|
data/lib/middleman-robots.rb
CHANGED
@@ -1,42 +1,27 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/blocks'
|
4
|
+
require 'middleman-robots/generators/sitemap_uri'
|
2
5
|
|
3
6
|
module Middleman
|
4
7
|
module Robots
|
5
8
|
# Robots Text Generator Class
|
6
9
|
class Generator
|
10
|
+
attr_accessor :rules, :sitemap_uri
|
11
|
+
|
7
12
|
def initialize(rules, sitemap_uri)
|
8
13
|
@rules = rules
|
9
14
|
@sitemap_uri = sitemap_uri
|
10
15
|
end
|
11
16
|
|
12
17
|
def process
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
blocks + "\n" + sitemap
|
18
|
-
elsif !blocks.empty?
|
19
|
-
blocks
|
20
|
-
elsif !sitemap.empty?
|
21
|
-
sitemap
|
22
|
-
else
|
23
|
-
''
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
def block_text
|
30
|
-
return '' if @rules.empty?
|
31
|
-
data = []
|
32
|
-
@rules.each do |rule|
|
33
|
-
data << Group.new(rule).text
|
34
|
-
end
|
35
|
-
data.join("\n")
|
36
|
-
end
|
18
|
+
text = [
|
19
|
+
Generators::Blocks.new(rules).text,
|
20
|
+
Generators::SitemapUri.new(sitemap_uri).text
|
21
|
+
].compact.join "\n\n"
|
37
22
|
|
38
|
-
|
39
|
-
|
23
|
+
text += "\n" if text.present?
|
24
|
+
text
|
40
25
|
end
|
41
26
|
end
|
42
27
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
# Block
|
9
|
+
#
|
10
|
+
# Generating Block in robots.txt
|
11
|
+
class Block
|
12
|
+
attr_accessor :rule
|
13
|
+
|
14
|
+
def initialize(rule)
|
15
|
+
@rule = rule
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
[
|
20
|
+
user_agent,
|
21
|
+
disallow,
|
22
|
+
allow
|
23
|
+
].compact.join("\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def user_agent
|
29
|
+
user_agent = rule[:user_agent].presence || rule['user-agent'].presence || '*'
|
30
|
+
if !user_agent.is_a?(String) && !user_agent.nil?
|
31
|
+
raise ArgumentError, '`user_agent` or `user-agent` option must be String or nil'
|
32
|
+
end
|
33
|
+
|
34
|
+
"User-Agent: #{user_agent}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def disallow
|
38
|
+
return nil if rule[:disallow].nil?
|
39
|
+
unless rule[:disallow].is_a? Array
|
40
|
+
raise ArgumentError, '`disallow` option must be Array or nil'
|
41
|
+
end
|
42
|
+
|
43
|
+
rule[:disallow].map { |path| "Disallow: #{File.join('/', path)}" }
|
44
|
+
end
|
45
|
+
|
46
|
+
def allow
|
47
|
+
return nil if rule[:allow].nil?
|
48
|
+
raise ArgumentError, '`allow` option must be Array or nil' unless rule[:allow].is_a? Array
|
49
|
+
|
50
|
+
rule[:allow].map { |path| "Allow: #{File.join('/', path)}" }
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
require 'middleman-robots/generators/block'
|
5
|
+
|
6
|
+
module Middleman
|
7
|
+
module Robots
|
8
|
+
module Generators
|
9
|
+
# Blocks
|
10
|
+
#
|
11
|
+
# Collection of ::Middleman::Robots::Generators::Block
|
12
|
+
class Blocks
|
13
|
+
def initialize(rules)
|
14
|
+
@rules = rules
|
15
|
+
@groups = @rules.map { |rule| Block.new(rule) } if @rules.present?
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
return nil if @groups.nil?
|
20
|
+
|
21
|
+
@groups.map(&:text).join "\n\n"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
class SitemapUri
|
9
|
+
attr_accessor :uri
|
10
|
+
|
11
|
+
def initialize(uri)
|
12
|
+
@uri = uri
|
13
|
+
end
|
14
|
+
|
15
|
+
def text
|
16
|
+
return nil if uri.blank?
|
17
|
+
raise ArgumentError, 'sitemap_uri must be string or nil' unless uri.is_a? String
|
18
|
+
|
19
|
+
"Sitemap: #{uri}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/middleman_extension.rb
CHANGED
data/middleman-robots.gemspec
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
lib = File.expand_path('lib', __dir__)
|
2
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
5
|
require 'middleman-robots/version'
|
@@ -16,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
16
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
20
|
spec.require_paths = ['lib']
|
19
|
-
spec.required_ruby_version = '>= 2.
|
21
|
+
spec.required_ruby_version = '>= 2.4.0'
|
20
22
|
|
21
23
|
spec.add_runtime_dependency 'middleman-cli', '>= 4.0'
|
22
24
|
spec.add_runtime_dependency 'middleman-core', '>= 4.0'
|
@@ -26,5 +28,6 @@ Gem::Specification.new do |spec|
|
|
26
28
|
spec.add_development_dependency 'capybara', '>= 2.18.0'
|
27
29
|
spec.add_development_dependency 'cucumber', '>= 3.1.0'
|
28
30
|
spec.add_development_dependency 'rake', '>= 12.3'
|
31
|
+
spec.add_development_dependency 'rspec'
|
29
32
|
spec.add_development_dependency 'rubocop', '>= 0.52.1'
|
30
33
|
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generator'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generator do
|
6
|
+
describe '#process' do
|
7
|
+
subject { Middleman::Robots::Generator.new(rules, sitemap_uri).process }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rules) do
|
11
|
+
[
|
12
|
+
{
|
13
|
+
user_agent: 'Googlebot',
|
14
|
+
disallow: %w[
|
15
|
+
tmp/*
|
16
|
+
/something/dir/file_disallow.html
|
17
|
+
],
|
18
|
+
allow: %w[
|
19
|
+
allow/*
|
20
|
+
/something/dir/file_allow.html
|
21
|
+
]
|
22
|
+
},
|
23
|
+
{
|
24
|
+
user_agent: 'Googlebot-Image',
|
25
|
+
disallow: %w[
|
26
|
+
tmp/*
|
27
|
+
/something/dir/file_disallow.html
|
28
|
+
],
|
29
|
+
allow: %w[
|
30
|
+
allow/*
|
31
|
+
/something/dir/file_allow.html
|
32
|
+
]
|
33
|
+
}
|
34
|
+
]
|
35
|
+
end
|
36
|
+
let(:sitemap_uri) { 'http://example.com/sitemap.xml' }
|
37
|
+
let(:expected) do
|
38
|
+
<<~ROBOTS
|
39
|
+
User-Agent: Googlebot
|
40
|
+
Disallow: /tmp/*
|
41
|
+
Disallow: /something/dir/file_disallow.html
|
42
|
+
Allow: /allow/*
|
43
|
+
Allow: /something/dir/file_allow.html
|
44
|
+
|
45
|
+
User-Agent: Googlebot-Image
|
46
|
+
Disallow: /tmp/*
|
47
|
+
Disallow: /something/dir/file_disallow.html
|
48
|
+
Allow: /allow/*
|
49
|
+
Allow: /something/dir/file_allow.html
|
50
|
+
|
51
|
+
Sitemap: http://example.com/sitemap.xml
|
52
|
+
ROBOTS
|
53
|
+
end
|
54
|
+
|
55
|
+
it { is_expected.to eq expected }
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'without options' do
|
59
|
+
let(:rules) { nil }
|
60
|
+
let(:sitemap_uri) { nil }
|
61
|
+
|
62
|
+
it { is_expected.to be_empty }
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,205 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/block'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::Block do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(rule).text }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rule) do
|
11
|
+
{
|
12
|
+
user_agent: 'GoogleBot',
|
13
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
14
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
let(:expected) do
|
18
|
+
expected = <<~ROBOTS
|
19
|
+
User-Agent: GoogleBot
|
20
|
+
Disallow: /tmp/*
|
21
|
+
Disallow: /someting/dir/disallow.html
|
22
|
+
Allow: /allow/*
|
23
|
+
Allow: /someting/dir/allow.html
|
24
|
+
ROBOTS
|
25
|
+
expected.chomp
|
26
|
+
end
|
27
|
+
|
28
|
+
it { is_expected.to eq expected }
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'without option' do
|
32
|
+
let(:rule) { {} }
|
33
|
+
|
34
|
+
it { is_expected.to eq 'User-Agent: *' }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with user_agent value is' do
|
38
|
+
context 'Middleman-Bot' do
|
39
|
+
let(:rule) { { user_agent: 'Middleman-Bot' } }
|
40
|
+
|
41
|
+
it { is_expected.to eq "User-Agent: #{rule[:user_agent]}" }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'nil' do
|
45
|
+
let(:rule) { { user_agent: nil } }
|
46
|
+
|
47
|
+
it { is_expected.to eq 'User-Agent: *' }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'empty' do
|
51
|
+
let(:rule) { { user_agent: '' } }
|
52
|
+
|
53
|
+
it { is_expected.to eq 'User-Agent: *' }
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'ERROR when' do
|
57
|
+
subject { -> { described_class.new(rule).text } }
|
58
|
+
|
59
|
+
context 'Array' do
|
60
|
+
let(:rule) { { user_agent: %w[a b] } }
|
61
|
+
|
62
|
+
it { is_expected.to raise_error ArgumentError }
|
63
|
+
end
|
64
|
+
|
65
|
+
context 'Numeric' do
|
66
|
+
let(:rule) { { user_agent: 1_000 } }
|
67
|
+
|
68
|
+
it { is_expected.to raise_error ArgumentError }
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'Symbol' do
|
72
|
+
let(:rule) { { user_agent: :user_agent } }
|
73
|
+
|
74
|
+
it { is_expected.to raise_error ArgumentError }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'with disallow value is' do
|
80
|
+
context 'single Array' do
|
81
|
+
let(:rule) { { disallow: %w[/tmp/*] } }
|
82
|
+
let(:expected) do
|
83
|
+
expected = <<~ROBOTS
|
84
|
+
User-Agent: *
|
85
|
+
Disallow: /tmp/*
|
86
|
+
ROBOTS
|
87
|
+
expected.chomp
|
88
|
+
end
|
89
|
+
|
90
|
+
it { is_expected.to eq expected }
|
91
|
+
end
|
92
|
+
|
93
|
+
context 'multiple Array' do
|
94
|
+
let(:rule) { { disallow: %w[/tmp/* /something/disallow.html] } }
|
95
|
+
let(:expected) do
|
96
|
+
expected = <<~ROBOTS
|
97
|
+
User-Agent: *
|
98
|
+
Disallow: /tmp/*
|
99
|
+
Disallow: /something/disallow.html
|
100
|
+
ROBOTS
|
101
|
+
expected.chomp
|
102
|
+
end
|
103
|
+
|
104
|
+
it { is_expected.to eq expected }
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'nil' do
|
108
|
+
let(:rule) { { disallow: nil } }
|
109
|
+
|
110
|
+
it { is_expected.to eq 'User-Agent: *' }
|
111
|
+
end
|
112
|
+
|
113
|
+
context 'ERROR when' do
|
114
|
+
subject { -> { described_class.new(rule).text } }
|
115
|
+
|
116
|
+
context 'empty' do
|
117
|
+
let(:rule) { { disallow: '' } }
|
118
|
+
|
119
|
+
it { is_expected.to raise_error ArgumentError }
|
120
|
+
end
|
121
|
+
|
122
|
+
context 'String' do
|
123
|
+
let(:rule) { { disallow: 'string' } }
|
124
|
+
|
125
|
+
it { is_expected.to raise_error ArgumentError }
|
126
|
+
end
|
127
|
+
|
128
|
+
context 'Numeric' do
|
129
|
+
let(:rule) { { disallow: 1_000 } }
|
130
|
+
|
131
|
+
it { is_expected.to raise_error ArgumentError }
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'Symbol' do
|
135
|
+
let(:rule) { { disallow: :disallow } }
|
136
|
+
|
137
|
+
it { is_expected.to raise_error ArgumentError }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context 'when allow value is' do
|
143
|
+
context 'single Array' do
|
144
|
+
let(:rule) { { allow: %w[/tmp/*] } }
|
145
|
+
let(:expected) do
|
146
|
+
expected = <<~ROBOTS
|
147
|
+
User-Agent: *
|
148
|
+
Allow: /tmp/*
|
149
|
+
ROBOTS
|
150
|
+
expected.chomp
|
151
|
+
end
|
152
|
+
|
153
|
+
it { is_expected.to eq expected }
|
154
|
+
end
|
155
|
+
|
156
|
+
context 'multiple Array' do
|
157
|
+
let(:rule) { { allow: %w[/tmp/* /something/allow.html] } }
|
158
|
+
let(:expected) do
|
159
|
+
expected = <<~ROBOTS
|
160
|
+
User-Agent: *
|
161
|
+
Allow: /tmp/*
|
162
|
+
Allow: /something/allow.html
|
163
|
+
ROBOTS
|
164
|
+
expected.chomp
|
165
|
+
end
|
166
|
+
|
167
|
+
it { is_expected.to eq expected }
|
168
|
+
end
|
169
|
+
|
170
|
+
context 'nil' do
|
171
|
+
let(:rule) { { allow: nil } }
|
172
|
+
|
173
|
+
it { is_expected.to eq 'User-Agent: *' }
|
174
|
+
end
|
175
|
+
|
176
|
+
context 'ERROR when' do
|
177
|
+
subject { -> { described_class.new(rule).text } }
|
178
|
+
|
179
|
+
context 'empty' do
|
180
|
+
let(:rule) { { allow: '' } }
|
181
|
+
|
182
|
+
it { is_expected.to raise_error ArgumentError }
|
183
|
+
end
|
184
|
+
|
185
|
+
context 'String' do
|
186
|
+
let(:rule) { { allow: 'string' } }
|
187
|
+
|
188
|
+
it { is_expected.to raise_error ArgumentError }
|
189
|
+
end
|
190
|
+
|
191
|
+
context 'Numeric' do
|
192
|
+
let(:rule) { { allow: 1_000 } }
|
193
|
+
|
194
|
+
it { is_expected.to raise_error ArgumentError }
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'Symbol' do
|
198
|
+
let(:rule) { { allow: :disallow } }
|
199
|
+
|
200
|
+
it { is_expected.to raise_error ArgumentError }
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/blocks'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::Blocks do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(rules).text }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rules) do
|
11
|
+
[
|
12
|
+
{
|
13
|
+
user_agent: 'GoogleBot',
|
14
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
15
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
16
|
+
},
|
17
|
+
{
|
18
|
+
user_agent: 'GoogleBot-Image',
|
19
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
20
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
21
|
+
}
|
22
|
+
]
|
23
|
+
end
|
24
|
+
let(:expected) do
|
25
|
+
expected = <<~ROBOTS
|
26
|
+
User-Agent: GoogleBot
|
27
|
+
Disallow: /tmp/*
|
28
|
+
Disallow: /someting/dir/disallow.html
|
29
|
+
Allow: /allow/*
|
30
|
+
Allow: /someting/dir/allow.html
|
31
|
+
|
32
|
+
User-Agent: GoogleBot-Image
|
33
|
+
Disallow: /tmp/*
|
34
|
+
Disallow: /someting/dir/disallow.html
|
35
|
+
Allow: /allow/*
|
36
|
+
Allow: /someting/dir/allow.html
|
37
|
+
ROBOTS
|
38
|
+
expected.chomp
|
39
|
+
end
|
40
|
+
|
41
|
+
it { is_expected.to eq expected }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'without option' do
|
45
|
+
let(:rules) { {} }
|
46
|
+
|
47
|
+
it { is_expected.to be_nil }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/sitemap_uri'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::SitemapUri do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(uri).text }
|
8
|
+
|
9
|
+
context 'with uri value is' do
|
10
|
+
context 'URL String' do
|
11
|
+
let(:uri) { 'https://example.com/sitemap.xml' }
|
12
|
+
|
13
|
+
it { is_expected.to eq "Sitemap: #{uri}" }
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'nil' do
|
17
|
+
let(:uri) { nil }
|
18
|
+
|
19
|
+
it { is_expected.to be_nil }
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'empty' do
|
23
|
+
let(:uri) { '' }
|
24
|
+
|
25
|
+
it { is_expected.to be_nil }
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'ERROR when' do
|
29
|
+
subject { -> { described_class.new(rule).text } }
|
30
|
+
|
31
|
+
context 'Array' do
|
32
|
+
let(:rule) { { uri: %w[a b] } }
|
33
|
+
|
34
|
+
it { is_expected.to raise_error ArgumentError }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'Numeric' do
|
38
|
+
let(:rule) { { uri: 1_000 } }
|
39
|
+
|
40
|
+
it { is_expected.to raise_error ArgumentError }
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'Symbol' do
|
44
|
+
let(:rule) { { uri: :uri } }
|
45
|
+
|
46
|
+
it { is_expected.to raise_error ArgumentError }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.configure do |config|
|
4
|
+
config.expect_with :rspec do |expectations|
|
5
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
6
|
+
end
|
7
|
+
|
8
|
+
config.mock_with :rspec do |mocks|
|
9
|
+
mocks.verify_partial_doubles = true
|
10
|
+
end
|
11
|
+
|
12
|
+
config.shared_context_metadata_behavior = :apply_to_host_groups
|
13
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: middleman-robots
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuya Matsushima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: middleman-cli
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '12.3'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: rubocop
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -131,6 +145,7 @@ extensions: []
|
|
131
145
|
extra_rdoc_files: []
|
132
146
|
files:
|
133
147
|
- ".gitignore"
|
148
|
+
- ".rspec"
|
134
149
|
- ".rubocop.yml"
|
135
150
|
- ".travis.yml"
|
136
151
|
- Gemfile
|
@@ -149,12 +164,17 @@ files:
|
|
149
164
|
- lib/middleman-robots.rb
|
150
165
|
- lib/middleman-robots/extension.rb
|
151
166
|
- lib/middleman-robots/generator.rb
|
152
|
-
- lib/middleman-robots/
|
167
|
+
- lib/middleman-robots/generators/block.rb
|
168
|
+
- lib/middleman-robots/generators/blocks.rb
|
169
|
+
- lib/middleman-robots/generators/sitemap_uri.rb
|
153
170
|
- lib/middleman-robots/version.rb
|
154
171
|
- lib/middleman_extension.rb
|
155
172
|
- middleman-robots.gemspec
|
156
|
-
-
|
157
|
-
-
|
173
|
+
- spec/lib/middleman-robots/generator_spec.rb
|
174
|
+
- spec/lib/middleman-robots/generators/block_spec.rb
|
175
|
+
- spec/lib/middleman-robots/generators/blocks_spec.rb
|
176
|
+
- spec/lib/middleman-robots/generators/sitemap_uri_spec.rb
|
177
|
+
- spec/spec_helper.rb
|
158
178
|
- tmp/robots/.gitkeep
|
159
179
|
homepage: https://github.com/yterajima/middleman-robots
|
160
180
|
licenses:
|
@@ -168,15 +188,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
168
188
|
requirements:
|
169
189
|
- - ">="
|
170
190
|
- !ruby/object:Gem::Version
|
171
|
-
version: 2.
|
191
|
+
version: 2.4.0
|
172
192
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
173
193
|
requirements:
|
174
194
|
- - ">="
|
175
195
|
- !ruby/object:Gem::Version
|
176
196
|
version: '0'
|
177
197
|
requirements: []
|
178
|
-
|
179
|
-
rubygems_version: 2.7.6
|
198
|
+
rubygems_version: 3.0.3
|
180
199
|
signing_key:
|
181
200
|
specification_version: 4
|
182
201
|
summary: Generate robots.txt by config.rb.
|
@@ -184,3 +203,8 @@ test_files:
|
|
184
203
|
- features/build.feature
|
185
204
|
- features/server.feature
|
186
205
|
- features/support/env.rb
|
206
|
+
- spec/lib/middleman-robots/generator_spec.rb
|
207
|
+
- spec/lib/middleman-robots/generators/block_spec.rb
|
208
|
+
- spec/lib/middleman-robots/generators/blocks_spec.rb
|
209
|
+
- spec/lib/middleman-robots/generators/sitemap_uri_spec.rb
|
210
|
+
- spec/spec_helper.rb
|
@@ -1,51 +0,0 @@
|
|
1
|
-
module Middleman
|
2
|
-
module Robots
|
3
|
-
# Robots Group Class
|
4
|
-
#
|
5
|
-
# Group class generate block in robots.txt
|
6
|
-
class Group
|
7
|
-
attr_reader :user_agent, :disallow, :allow
|
8
|
-
|
9
|
-
def initialize(rule)
|
10
|
-
@user_agent = generate_user_agent(rule)
|
11
|
-
@disallow = generate_disallow(rule)
|
12
|
-
@allow = generate_allow(rule)
|
13
|
-
end
|
14
|
-
|
15
|
-
def text
|
16
|
-
group = []
|
17
|
-
group << "User-Agent: #{@user_agent}" unless @user_agent.empty?
|
18
|
-
group << @disallow.collect { |item| "Disallow: #{item}" }.join("\n") if @disallow.length.positive?
|
19
|
-
group << @allow.collect { |item| "Allow: #{item}" }.join("\n") if @allow.length.positive?
|
20
|
-
group.join("\n") + "\n"
|
21
|
-
end
|
22
|
-
|
23
|
-
private
|
24
|
-
|
25
|
-
def generate_user_agent(rule)
|
26
|
-
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
27
|
-
rule[:user_agent] || rule['user-agent']
|
28
|
-
end
|
29
|
-
|
30
|
-
def generate_disallow(rule)
|
31
|
-
paths = []
|
32
|
-
return paths unless rule.key?(:disallow)
|
33
|
-
|
34
|
-
rule[:disallow].each do |path|
|
35
|
-
paths << File.join('/', path)
|
36
|
-
end
|
37
|
-
paths
|
38
|
-
end
|
39
|
-
|
40
|
-
def generate_allow(rule)
|
41
|
-
paths = []
|
42
|
-
return paths unless rule.key?(:allow)
|
43
|
-
|
44
|
-
rule[:allow].each do |path|
|
45
|
-
paths << File.join('/', path)
|
46
|
-
end
|
47
|
-
paths
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
data/tests/test_generator.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'minitest/autorun'
|
2
|
-
require_relative '../lib/middleman-robots/generator.rb'
|
3
|
-
|
4
|
-
class TestGenerator < MiniTest::Test
|
5
|
-
def test_process
|
6
|
-
sitemap_uri = 'http://example.com/sitemap.xml'
|
7
|
-
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
8
|
-
assert_equal expected, generator.process
|
9
|
-
end
|
10
|
-
|
11
|
-
private
|
12
|
-
|
13
|
-
def rules
|
14
|
-
[
|
15
|
-
{
|
16
|
-
user_agent: 'Googlebot',
|
17
|
-
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
18
|
-
allow: %w[allow/* /something/dir/file_allow.html]
|
19
|
-
},
|
20
|
-
{
|
21
|
-
user_agent: 'Googlebot-Image',
|
22
|
-
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
23
|
-
allow: %w[allow/* /something/dir/file_allow.html]
|
24
|
-
}
|
25
|
-
]
|
26
|
-
end
|
27
|
-
|
28
|
-
def expected
|
29
|
-
"User-Agent: Googlebot
|
30
|
-
Disallow: /tmp/*
|
31
|
-
Disallow: /something/dir/file_disallow.html
|
32
|
-
Allow: /allow/*
|
33
|
-
Allow: /something/dir/file_allow.html
|
34
|
-
|
35
|
-
User-Agent: Googlebot-Image
|
36
|
-
Disallow: /tmp/*
|
37
|
-
Disallow: /something/dir/file_disallow.html
|
38
|
-
Allow: /allow/*
|
39
|
-
Allow: /something/dir/file_allow.html
|
40
|
-
|
41
|
-
Sitemap: http://example.com/sitemap.xml"
|
42
|
-
end
|
43
|
-
end
|
data/tests/test_group.rb
DELETED
@@ -1,120 +0,0 @@
|
|
1
|
-
require 'minitest/autorun'
|
2
|
-
require_relative '../lib/middleman-robots/group.rb'
|
3
|
-
|
4
|
-
class TestGroup < MiniTest::Test
|
5
|
-
def test_initialize
|
6
|
-
rule = {
|
7
|
-
user_agent: 'GoogleBot',
|
8
|
-
disallow: %w[tmp/* /someting/dir/disallow.html],
|
9
|
-
allow: %w[allow/* /someting/dir/allow.html]
|
10
|
-
}
|
11
|
-
group = Middleman::Robots::Group.new(rule)
|
12
|
-
|
13
|
-
assert_equal rule[:user_agent], group.user_agent
|
14
|
-
assert_equal %w[/tmp/* /someting/dir/disallow.html], group.disallow
|
15
|
-
assert_equal %w[/allow/* /someting/dir/allow.html], group.allow
|
16
|
-
end
|
17
|
-
|
18
|
-
def test_initialize_without_user_agent
|
19
|
-
rule = {
|
20
|
-
disallow: %w[/tmp/*],
|
21
|
-
allow: %w[/allow/*]
|
22
|
-
}
|
23
|
-
group = Middleman::Robots::Group.new(rule)
|
24
|
-
|
25
|
-
assert_equal '*', group.user_agent
|
26
|
-
assert_equal rule[:disallow], group.disallow
|
27
|
-
assert_equal rule[:allow], group.allow
|
28
|
-
end
|
29
|
-
|
30
|
-
def test_initialize_without_disallow
|
31
|
-
rule = {
|
32
|
-
user_agent: 'GoogleBot',
|
33
|
-
allow: %w[/allow/* /someting/dir/allow.html]
|
34
|
-
}
|
35
|
-
group = Middleman::Robots::Group.new(rule)
|
36
|
-
|
37
|
-
assert_equal rule[:user_agent], group.user_agent
|
38
|
-
assert_equal [], group.disallow
|
39
|
-
assert_equal rule[:allow], group.allow
|
40
|
-
end
|
41
|
-
|
42
|
-
def test_initialize_without_allow
|
43
|
-
rule = {
|
44
|
-
user_agent: 'GoogleBot',
|
45
|
-
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
46
|
-
}
|
47
|
-
group = Middleman::Robots::Group.new(rule)
|
48
|
-
|
49
|
-
assert_equal rule[:user_agent], group.user_agent
|
50
|
-
assert_equal rule[:disallow], group.disallow
|
51
|
-
assert_equal [], group.allow
|
52
|
-
end
|
53
|
-
|
54
|
-
def test_text
|
55
|
-
rule = {
|
56
|
-
user_agent: 'GoogleBot',
|
57
|
-
disallow: %w[tmp/* /someting/dir/disallow.html],
|
58
|
-
allow: %w[allow/* /someting/dir/allow.html]
|
59
|
-
}
|
60
|
-
group = Middleman::Robots::Group.new(rule)
|
61
|
-
|
62
|
-
expected = <<~ROBOTS
|
63
|
-
User-Agent: GoogleBot
|
64
|
-
Disallow: /tmp/*
|
65
|
-
Disallow: /someting/dir/disallow.html
|
66
|
-
Allow: /allow/*
|
67
|
-
Allow: /someting/dir/allow.html
|
68
|
-
ROBOTS
|
69
|
-
|
70
|
-
assert_equal expected, group.text
|
71
|
-
end
|
72
|
-
|
73
|
-
def test_text_without_user_agent
|
74
|
-
rule = {
|
75
|
-
disallow: %w[/tmp/*],
|
76
|
-
allow: %w[/allow/*]
|
77
|
-
}
|
78
|
-
group = Middleman::Robots::Group.new(rule)
|
79
|
-
|
80
|
-
expected = <<~ROBOTS
|
81
|
-
User-Agent: *
|
82
|
-
Disallow: /tmp/*
|
83
|
-
Allow: /allow/*
|
84
|
-
ROBOTS
|
85
|
-
|
86
|
-
assert_equal expected, group.text
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_text_without_disallow
|
90
|
-
rule = {
|
91
|
-
user_agent: 'GoogleBot',
|
92
|
-
allow: %w[/allow/* /someting/dir/allow.html]
|
93
|
-
}
|
94
|
-
group = Middleman::Robots::Group.new(rule)
|
95
|
-
|
96
|
-
expected = <<~ROBOTS
|
97
|
-
User-Agent: GoogleBot
|
98
|
-
Allow: /allow/*
|
99
|
-
Allow: /someting/dir/allow.html
|
100
|
-
ROBOTS
|
101
|
-
|
102
|
-
assert_equal expected, group.text
|
103
|
-
end
|
104
|
-
|
105
|
-
def test_text_without_allow
|
106
|
-
rule = {
|
107
|
-
user_agent: 'GoogleBot',
|
108
|
-
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
109
|
-
}
|
110
|
-
group = Middleman::Robots::Group.new(rule)
|
111
|
-
|
112
|
-
expected = <<~ROBOTS
|
113
|
-
User-Agent: GoogleBot
|
114
|
-
Disallow: /tmp/*
|
115
|
-
Disallow: /someting/dir/disallow.html
|
116
|
-
ROBOTS
|
117
|
-
|
118
|
-
assert_equal expected, group.text
|
119
|
-
end
|
120
|
-
end
|