middleman-robots 1.3.2 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +25 -0
- data/.rspec +2 -0
- data/.rubocop.yml +28 -0
- data/.travis.yml +8 -9
- data/Gemfile +2 -0
- data/README.md +16 -16
- data/Rakefile +3 -3
- data/cucumber.yml +1 -0
- data/features/build.feature +30 -39
- data/features/support/env.rb +3 -2
- data/fixtures/server-app/config.rb +28 -14
- data/lib/middleman-robots.rb +4 -2
- data/lib/middleman-robots/extension.rb +13 -8
- data/lib/middleman-robots/generator.rb +12 -26
- data/lib/middleman-robots/generators/block.rb +55 -0
- data/lib/middleman-robots/generators/blocks.rb +26 -0
- data/lib/middleman-robots/generators/sitemap_uri.rb +24 -0
- data/lib/middleman-robots/version.rb +3 -1
- data/lib/middleman_extension.rb +2 -1
- data/middleman-robots.gemspec +21 -18
- data/spec/lib/middleman-robots/generator_spec.rb +65 -0
- data/spec/lib/middleman-robots/generators/block_spec.rb +205 -0
- data/spec/lib/middleman-robots/generators/blocks_spec.rb +50 -0
- data/spec/lib/middleman-robots/generators/sitemap_uri_spec.rb +51 -0
- data/spec/spec_helper.rb +13 -0
- metadata +97 -29
- data/lib/middleman-robots/group.rb +0 -50
- data/tests/test_generator.rb +0 -37
- data/tests/test_group.rb +0 -121
@@ -1,42 +1,28 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/blocks'
|
4
|
+
require 'middleman-robots/generators/sitemap_uri'
|
2
5
|
|
3
6
|
module Middleman
|
4
7
|
module Robots
|
5
8
|
# Robots Text Generator Class
|
6
9
|
class Generator
|
10
|
+
attr_accessor :rules, :sitemap_uri
|
11
|
+
|
7
12
|
def initialize(rules, sitemap_uri)
|
8
13
|
@rules = rules
|
9
14
|
@sitemap_uri = sitemap_uri
|
10
15
|
end
|
11
16
|
|
12
17
|
def process
|
13
|
-
|
14
|
-
|
18
|
+
text = [
|
19
|
+
Generators::Blocks.new(rules).text,
|
20
|
+
Generators::SitemapUri.new(sitemap_uri).text
|
21
|
+
].compact.join "\n\n"
|
15
22
|
|
16
|
-
|
17
|
-
|
18
|
-
elsif !blocks.empty?
|
19
|
-
blocks
|
20
|
-
elsif !sitemap.empty?
|
21
|
-
sitemap
|
22
|
-
else
|
23
|
-
''
|
24
|
-
end
|
23
|
+
text += "\n" if text.present?
|
24
|
+
text
|
25
25
|
end
|
26
|
-
|
27
|
-
private
|
28
|
-
def block_text
|
29
|
-
return '' if @rules.empty?
|
30
|
-
data = []
|
31
|
-
@rules.each do |rule|
|
32
|
-
data << Group.new(rule).text
|
33
|
-
end
|
34
|
-
data.join("\n")
|
35
|
-
end
|
36
|
-
|
37
|
-
def sitemap_text
|
38
|
-
@sitemap_uri ? "Sitemap: #{@sitemap_uri}" : ''
|
39
|
-
end
|
40
26
|
end
|
41
27
|
end
|
42
28
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
# Block
|
9
|
+
#
|
10
|
+
# Generating Block in robots.txt
|
11
|
+
class Block
|
12
|
+
attr_accessor :rule
|
13
|
+
|
14
|
+
def initialize(rule)
|
15
|
+
@rule = rule
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
[
|
20
|
+
user_agent,
|
21
|
+
disallow,
|
22
|
+
allow
|
23
|
+
].compact.join("\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def user_agent
|
29
|
+
user_agent = rule[:user_agent].presence || rule['user-agent'].presence || '*'
|
30
|
+
if !user_agent.is_a?(String) && !user_agent.nil?
|
31
|
+
raise ArgumentError, '`user_agent` or `user-agent` option must be String or nil'
|
32
|
+
end
|
33
|
+
|
34
|
+
"User-Agent: #{user_agent}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def disallow
|
38
|
+
return nil if rule[:disallow].nil?
|
39
|
+
unless rule[:disallow].is_a? Array
|
40
|
+
raise ArgumentError, '`disallow` option must be Array or nil'
|
41
|
+
end
|
42
|
+
|
43
|
+
rule[:disallow].map { |path| "Disallow: #{File.join('/', path)}" }
|
44
|
+
end
|
45
|
+
|
46
|
+
def allow
|
47
|
+
return nil if rule[:allow].nil?
|
48
|
+
raise ArgumentError, '`allow` option must be Array or nil' unless rule[:allow].is_a? Array
|
49
|
+
|
50
|
+
rule[:allow].map { |path| "Allow: #{File.join('/', path)}" }
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
require 'middleman-robots/generators/block'
|
5
|
+
|
6
|
+
module Middleman
|
7
|
+
module Robots
|
8
|
+
module Generators
|
9
|
+
# Blocks
|
10
|
+
#
|
11
|
+
# Collection of ::Middleman::Robots::Generators::Block
|
12
|
+
class Blocks
|
13
|
+
def initialize(rules)
|
14
|
+
@rules = rules
|
15
|
+
@groups = @rules.map { |rule| Block.new(rule) } if @rules.present?
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
return nil if @groups.nil?
|
20
|
+
|
21
|
+
@groups.map(&:text).join "\n\n"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
class SitemapUri
|
9
|
+
attr_accessor :uri
|
10
|
+
|
11
|
+
def initialize(uri)
|
12
|
+
@uri = uri
|
13
|
+
end
|
14
|
+
|
15
|
+
def text
|
16
|
+
return nil if uri.blank?
|
17
|
+
raise ArgumentError, 'sitemap_uri must be string or nil' unless uri.is_a? String
|
18
|
+
|
19
|
+
"Sitemap: #{uri}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/middleman_extension.rb
CHANGED
data/middleman-robots.gemspec
CHANGED
@@ -1,30 +1,33 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'middleman-robots/version'
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
+
spec.name = 'middleman-robots'
|
8
9
|
spec.version = Middleman::Robots::VERSION
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
14
|
-
spec.license =
|
10
|
+
spec.authors = ['Yuya Matsushima']
|
11
|
+
spec.email = ['terra@e2esound.com']
|
12
|
+
spec.summary = 'Generate robots.txt by config.rb.'
|
13
|
+
spec.description = 'middleman-robots create robots.txt includes Allow or Disallow and sitemap path.'
|
14
|
+
spec.homepage = 'https://github.com/yterajima/middleman-robots'
|
15
|
+
spec.license = 'MIT'
|
15
16
|
|
16
17
|
spec.files = `git ls-files -z`.split("\x0")
|
17
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
-
spec.require_paths = [
|
20
|
-
spec.required_ruby_version = '>= 2.
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
spec.required_ruby_version = '>= 2.6.0'
|
21
22
|
|
22
|
-
spec.add_runtime_dependency
|
23
|
+
spec.add_runtime_dependency 'middleman-cli', '>= 4.0', '<= 4.4'
|
24
|
+
spec.add_runtime_dependency 'middleman-core', '>= 4.0', '<= 4.4'
|
23
25
|
|
24
|
-
spec.add_development_dependency
|
25
|
-
spec.add_development_dependency
|
26
|
-
spec.add_development_dependency
|
27
|
-
spec.add_development_dependency
|
28
|
-
spec.add_development_dependency
|
26
|
+
spec.add_development_dependency 'aruba', '>= 0.14.3'
|
27
|
+
spec.add_development_dependency 'bundler', '>= 1.16'
|
28
|
+
spec.add_development_dependency 'capybara', '>= 2.18.0'
|
29
|
+
spec.add_development_dependency 'cucumber', '>= 3.1.0'
|
30
|
+
spec.add_development_dependency 'rake', '>= 12.3'
|
31
|
+
spec.add_development_dependency 'rspec'
|
32
|
+
spec.add_development_dependency 'rubocop', '>= 0.52.1'
|
29
33
|
end
|
30
|
-
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generator'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generator do
|
6
|
+
describe '#process' do
|
7
|
+
subject { Middleman::Robots::Generator.new(rules, sitemap_uri).process }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rules) do
|
11
|
+
[
|
12
|
+
{
|
13
|
+
user_agent: 'Googlebot',
|
14
|
+
disallow: %w[
|
15
|
+
tmp/*
|
16
|
+
/something/dir/file_disallow.html
|
17
|
+
],
|
18
|
+
allow: %w[
|
19
|
+
allow/*
|
20
|
+
/something/dir/file_allow.html
|
21
|
+
]
|
22
|
+
},
|
23
|
+
{
|
24
|
+
user_agent: 'Googlebot-Image',
|
25
|
+
disallow: %w[
|
26
|
+
tmp/*
|
27
|
+
/something/dir/file_disallow.html
|
28
|
+
],
|
29
|
+
allow: %w[
|
30
|
+
allow/*
|
31
|
+
/something/dir/file_allow.html
|
32
|
+
]
|
33
|
+
}
|
34
|
+
]
|
35
|
+
end
|
36
|
+
let(:sitemap_uri) { 'http://example.com/sitemap.xml' }
|
37
|
+
let(:expected) do
|
38
|
+
<<~ROBOTS
|
39
|
+
User-Agent: Googlebot
|
40
|
+
Disallow: /tmp/*
|
41
|
+
Disallow: /something/dir/file_disallow.html
|
42
|
+
Allow: /allow/*
|
43
|
+
Allow: /something/dir/file_allow.html
|
44
|
+
|
45
|
+
User-Agent: Googlebot-Image
|
46
|
+
Disallow: /tmp/*
|
47
|
+
Disallow: /something/dir/file_disallow.html
|
48
|
+
Allow: /allow/*
|
49
|
+
Allow: /something/dir/file_allow.html
|
50
|
+
|
51
|
+
Sitemap: http://example.com/sitemap.xml
|
52
|
+
ROBOTS
|
53
|
+
end
|
54
|
+
|
55
|
+
it { is_expected.to eq expected }
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'without options' do
|
59
|
+
let(:rules) { nil }
|
60
|
+
let(:sitemap_uri) { nil }
|
61
|
+
|
62
|
+
it { is_expected.to be_empty }
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,205 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/block'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::Block do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(rule).text }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rule) do
|
11
|
+
{
|
12
|
+
user_agent: 'GoogleBot',
|
13
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
14
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
let(:expected) do
|
18
|
+
expected = <<~ROBOTS
|
19
|
+
User-Agent: GoogleBot
|
20
|
+
Disallow: /tmp/*
|
21
|
+
Disallow: /someting/dir/disallow.html
|
22
|
+
Allow: /allow/*
|
23
|
+
Allow: /someting/dir/allow.html
|
24
|
+
ROBOTS
|
25
|
+
expected.chomp
|
26
|
+
end
|
27
|
+
|
28
|
+
it { is_expected.to eq expected }
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'without option' do
|
32
|
+
let(:rule) { {} }
|
33
|
+
|
34
|
+
it { is_expected.to eq 'User-Agent: *' }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with user_agent value is' do
|
38
|
+
context 'Middleman-Bot' do
|
39
|
+
let(:rule) { { user_agent: 'Middleman-Bot' } }
|
40
|
+
|
41
|
+
it { is_expected.to eq "User-Agent: #{rule[:user_agent]}" }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'nil' do
|
45
|
+
let(:rule) { { user_agent: nil } }
|
46
|
+
|
47
|
+
it { is_expected.to eq 'User-Agent: *' }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'empty' do
|
51
|
+
let(:rule) { { user_agent: '' } }
|
52
|
+
|
53
|
+
it { is_expected.to eq 'User-Agent: *' }
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'ERROR when' do
|
57
|
+
subject { -> { described_class.new(rule).text } }
|
58
|
+
|
59
|
+
context 'Array' do
|
60
|
+
let(:rule) { { user_agent: %w[a b] } }
|
61
|
+
|
62
|
+
it { is_expected.to raise_error ArgumentError }
|
63
|
+
end
|
64
|
+
|
65
|
+
context 'Numeric' do
|
66
|
+
let(:rule) { { user_agent: 1_000 } }
|
67
|
+
|
68
|
+
it { is_expected.to raise_error ArgumentError }
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'Symbol' do
|
72
|
+
let(:rule) { { user_agent: :user_agent } }
|
73
|
+
|
74
|
+
it { is_expected.to raise_error ArgumentError }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'with disallow value is' do
|
80
|
+
context 'single Array' do
|
81
|
+
let(:rule) { { disallow: %w[/tmp/*] } }
|
82
|
+
let(:expected) do
|
83
|
+
expected = <<~ROBOTS
|
84
|
+
User-Agent: *
|
85
|
+
Disallow: /tmp/*
|
86
|
+
ROBOTS
|
87
|
+
expected.chomp
|
88
|
+
end
|
89
|
+
|
90
|
+
it { is_expected.to eq expected }
|
91
|
+
end
|
92
|
+
|
93
|
+
context 'multiple Array' do
|
94
|
+
let(:rule) { { disallow: %w[/tmp/* /something/disallow.html] } }
|
95
|
+
let(:expected) do
|
96
|
+
expected = <<~ROBOTS
|
97
|
+
User-Agent: *
|
98
|
+
Disallow: /tmp/*
|
99
|
+
Disallow: /something/disallow.html
|
100
|
+
ROBOTS
|
101
|
+
expected.chomp
|
102
|
+
end
|
103
|
+
|
104
|
+
it { is_expected.to eq expected }
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'nil' do
|
108
|
+
let(:rule) { { disallow: nil } }
|
109
|
+
|
110
|
+
it { is_expected.to eq 'User-Agent: *' }
|
111
|
+
end
|
112
|
+
|
113
|
+
context 'ERROR when' do
|
114
|
+
subject { -> { described_class.new(rule).text } }
|
115
|
+
|
116
|
+
context 'empty' do
|
117
|
+
let(:rule) { { disallow: '' } }
|
118
|
+
|
119
|
+
it { is_expected.to raise_error ArgumentError }
|
120
|
+
end
|
121
|
+
|
122
|
+
context 'String' do
|
123
|
+
let(:rule) { { disallow: 'string' } }
|
124
|
+
|
125
|
+
it { is_expected.to raise_error ArgumentError }
|
126
|
+
end
|
127
|
+
|
128
|
+
context 'Numeric' do
|
129
|
+
let(:rule) { { disallow: 1_000 } }
|
130
|
+
|
131
|
+
it { is_expected.to raise_error ArgumentError }
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'Symbol' do
|
135
|
+
let(:rule) { { disallow: :disallow } }
|
136
|
+
|
137
|
+
it { is_expected.to raise_error ArgumentError }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context 'when allow value is' do
|
143
|
+
context 'single Array' do
|
144
|
+
let(:rule) { { allow: %w[/tmp/*] } }
|
145
|
+
let(:expected) do
|
146
|
+
expected = <<~ROBOTS
|
147
|
+
User-Agent: *
|
148
|
+
Allow: /tmp/*
|
149
|
+
ROBOTS
|
150
|
+
expected.chomp
|
151
|
+
end
|
152
|
+
|
153
|
+
it { is_expected.to eq expected }
|
154
|
+
end
|
155
|
+
|
156
|
+
context 'multiple Array' do
|
157
|
+
let(:rule) { { allow: %w[/tmp/* /something/allow.html] } }
|
158
|
+
let(:expected) do
|
159
|
+
expected = <<~ROBOTS
|
160
|
+
User-Agent: *
|
161
|
+
Allow: /tmp/*
|
162
|
+
Allow: /something/allow.html
|
163
|
+
ROBOTS
|
164
|
+
expected.chomp
|
165
|
+
end
|
166
|
+
|
167
|
+
it { is_expected.to eq expected }
|
168
|
+
end
|
169
|
+
|
170
|
+
context 'nil' do
|
171
|
+
let(:rule) { { allow: nil } }
|
172
|
+
|
173
|
+
it { is_expected.to eq 'User-Agent: *' }
|
174
|
+
end
|
175
|
+
|
176
|
+
context 'ERROR when' do
|
177
|
+
subject { -> { described_class.new(rule).text } }
|
178
|
+
|
179
|
+
context 'empty' do
|
180
|
+
let(:rule) { { allow: '' } }
|
181
|
+
|
182
|
+
it { is_expected.to raise_error ArgumentError }
|
183
|
+
end
|
184
|
+
|
185
|
+
context 'String' do
|
186
|
+
let(:rule) { { allow: 'string' } }
|
187
|
+
|
188
|
+
it { is_expected.to raise_error ArgumentError }
|
189
|
+
end
|
190
|
+
|
191
|
+
context 'Numeric' do
|
192
|
+
let(:rule) { { allow: 1_000 } }
|
193
|
+
|
194
|
+
it { is_expected.to raise_error ArgumentError }
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'Symbol' do
|
198
|
+
let(:rule) { { allow: :disallow } }
|
199
|
+
|
200
|
+
it { is_expected.to raise_error ArgumentError }
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|