middleman-robots 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rubocop.yml +38 -0
- data/.travis.yml +7 -5
- data/README.md +15 -15
- data/features/build.feature +30 -38
- data/features/support/env.rb +0 -1
- data/fixtures/server-app/config.rb +14 -14
- data/lib/middleman-robots.rb +2 -2
- data/lib/middleman-robots/extension.rb +11 -5
- data/lib/middleman-robots/generator.rb +11 -10
- data/lib/middleman-robots/group.rb +21 -20
- data/lib/middleman-robots/version.rb +1 -1
- data/lib/middleman_extension.rb +1 -2
- data/middleman-robots.gemspec +16 -17
- data/tests/test_generator.rb +17 -11
- data/tests/test_group.rb +37 -38
- metadata +37 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: b1f3a8305157e44491560743f0aff0ea58b50bf46f5eb10c61482cea721d064b
|
4
|
+
data.tar.gz: 5ffa930584f52dc32fdbbe9ed87edc58566ab560ca5357499d2323409dbe6aa2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb58e902248a63782ddd07779b963a0b9f0baa742c3bb109386577c7942be9efb23f63f201b60aecebd0cf02fea6340d7ba5305a84607f47bbef1b2d34c9d04d
|
7
|
+
data.tar.gz: '08205a13c77a299db354736153d059b40bc27ce4167b719d93bda74a3580cfe881ec39da91d2adab6d9f30a7fa88765e591584c8c8fc695713541b0059cfc7c8'
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
AllCops:
|
2
|
+
Exclude:
|
3
|
+
- 'pkg/*'
|
4
|
+
- 'tmp/**/*'
|
5
|
+
- 'vendor/**/*'
|
6
|
+
TargetRubyVersion: 2.5
|
7
|
+
DisplayCopNames: true
|
8
|
+
|
9
|
+
Rails:
|
10
|
+
Enabled: false
|
11
|
+
|
12
|
+
Style/ClassAndModuleChildren:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
Style/Documentation:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Style/FrozenStringLiteralComment:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/RedundantSelf:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Gemspec/RequiredRubyVersion:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Metrics/LineLength:
|
28
|
+
Max: 120
|
29
|
+
Exclude:
|
30
|
+
- '*.gemspec'
|
31
|
+
|
32
|
+
Metrics/MethodLength:
|
33
|
+
Max: 20
|
34
|
+
|
35
|
+
Naming/FileName:
|
36
|
+
Enabled: true
|
37
|
+
Exclude:
|
38
|
+
- 'lib/middleman-robots.rb'
|
data/.travis.yml
CHANGED
@@ -5,16 +5,18 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.
|
9
|
-
- 2.3
|
10
|
-
- 2.
|
8
|
+
- 2.5.0
|
9
|
+
- 2.4.3
|
10
|
+
- 2.3.6
|
11
11
|
os:
|
12
12
|
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
|
-
- rvm:
|
17
|
-
script:
|
16
|
+
- rvm: ruby-head
|
17
|
+
script:
|
18
|
+
- bundle exec rake test
|
19
|
+
- bundle exec rubocop
|
18
20
|
cache: bundler
|
19
21
|
env: TEST=true
|
20
22
|
notifications:
|
data/README.md
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
[](http://badge.fury.io/rb/middleman-robots)
|
4
4
|
[](https://travis-ci.org/yterajima/middleman-robots)
|
5
5
|
|
6
|
-
`middleman-robots` is an extension of [Middleman](
|
6
|
+
`middleman-robots` is an extension of [Middleman](https://middlemanapp.com/). This can create `robots.txt`.
|
7
7
|
|
8
8
|
This plugin support Middleman v3-stable and v4.
|
9
9
|
|
@@ -26,10 +26,10 @@ Or install it yourself as:
|
|
26
26
|
```ruby
|
27
27
|
# config.rb
|
28
28
|
activate :robots,
|
29
|
-
:
|
30
|
-
{:
|
29
|
+
rules: [
|
30
|
+
{ user_agent: '*', allow: %w[/] }
|
31
31
|
],
|
32
|
-
:
|
32
|
+
sitemap: 'https://example.com/sitemap.xml'
|
33
33
|
```
|
34
34
|
|
35
35
|
Created `robots.txt`:
|
@@ -38,28 +38,28 @@ Created `robots.txt`:
|
|
38
38
|
User-Agent: *
|
39
39
|
Allow: /
|
40
40
|
|
41
|
-
Sitemap:
|
41
|
+
Sitemap: https://example.com/sitemap.xml
|
42
42
|
|
43
43
|
```
|
44
44
|
|
45
|
-
You can use options,
|
45
|
+
You can use options, `rules` [ {`user_agent`(string), `allow`(array), `disallow`(array)}] and `sitemap`. Like this:
|
46
46
|
|
47
47
|
```ruby
|
48
48
|
# config.rb
|
49
49
|
activate :robots,
|
50
|
-
:
|
50
|
+
rules: [
|
51
51
|
{
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
52
|
+
user_agent: 'Googlebot',
|
53
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
54
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
55
55
|
},
|
56
56
|
{
|
57
|
-
:
|
58
|
-
:
|
59
|
-
:
|
57
|
+
user_agent: 'Googlebot-Image',
|
58
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
59
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
60
60
|
}
|
61
61
|
],
|
62
|
-
:
|
62
|
+
sitemap: 'https://example.com/sitemap.xml'
|
63
63
|
```
|
64
64
|
|
65
65
|
Created `robots.txt`:
|
@@ -77,7 +77,7 @@ Disallow: /something/dir/file_disallow.html
|
|
77
77
|
Allow: /allow/
|
78
78
|
Allow: /something/dir/file_allow.html
|
79
79
|
|
80
|
-
Sitemap:
|
80
|
+
Sitemap: https://example.com/sitemap.xml
|
81
81
|
|
82
82
|
```
|
83
83
|
|
data/features/build.feature
CHANGED
@@ -18,9 +18,7 @@ Feature: Middleman-Robots on build
|
|
18
18
|
Given a fixture app "basic-app"
|
19
19
|
And a file named "config.rb" with:
|
20
20
|
"""
|
21
|
-
activate :robots, :
|
22
|
-
{:user_agent => '*'}
|
23
|
-
]
|
21
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
24
22
|
"""
|
25
23
|
And a successfully built app at "basic-app"
|
26
24
|
When I cd to "build"
|
@@ -34,9 +32,7 @@ Feature: Middleman-Robots on build
|
|
34
32
|
And a file named "config.rb" with:
|
35
33
|
"""
|
36
34
|
activate :robots do |r|
|
37
|
-
r.rules = [
|
38
|
-
{:user_agent => '*'}
|
39
|
-
]
|
35
|
+
r.rules = [ { user_agent: '*'} ]
|
40
36
|
end
|
41
37
|
"""
|
42
38
|
And a successfully built app at "basic-app"
|
@@ -50,9 +46,7 @@ Feature: Middleman-Robots on build
|
|
50
46
|
Given a fixture app "basic-app"
|
51
47
|
And a file named "config.rb" with:
|
52
48
|
"""
|
53
|
-
activate :robots, :
|
54
|
-
{:user_agent => '*'}
|
55
|
-
]
|
49
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
56
50
|
"""
|
57
51
|
And a successfully built app at "basic-app"
|
58
52
|
When I cd to "build"
|
@@ -65,9 +59,7 @@ Feature: Middleman-Robots on build
|
|
65
59
|
Given a fixture app "basic-app"
|
66
60
|
And a file named "config.rb" with:
|
67
61
|
"""
|
68
|
-
activate :robots, :
|
69
|
-
{'user-agent' => '*'}
|
70
|
-
]
|
62
|
+
activate :robots, rules: [ { 'user-agent': '*' } ]
|
71
63
|
"""
|
72
64
|
And a successfully built app at "basic-app"
|
73
65
|
When I cd to "build"
|
@@ -80,10 +72,10 @@ Feature: Middleman-Robots on build
|
|
80
72
|
Given a fixture app "basic-app"
|
81
73
|
And a file named "config.rb" with:
|
82
74
|
"""
|
83
|
-
activate :robots, :
|
75
|
+
activate :robots, rules: [
|
84
76
|
{
|
85
|
-
:
|
86
|
-
:
|
77
|
+
user_agent: '*',
|
78
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html]
|
87
79
|
}
|
88
80
|
]
|
89
81
|
"""
|
@@ -100,10 +92,10 @@ Feature: Middleman-Robots on build
|
|
100
92
|
Given a fixture app "basic-app"
|
101
93
|
And a file named "config.rb" with:
|
102
94
|
"""
|
103
|
-
activate :robots, :
|
95
|
+
activate :robots, rules: [
|
104
96
|
{
|
105
|
-
:
|
106
|
-
:
|
97
|
+
user_agent: '*',
|
98
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
107
99
|
}
|
108
100
|
]
|
109
101
|
"""
|
@@ -120,11 +112,11 @@ Feature: Middleman-Robots on build
|
|
120
112
|
Given a fixture app "basic-app"
|
121
113
|
And a file named "config.rb" with:
|
122
114
|
"""
|
123
|
-
activate :robots, :
|
115
|
+
activate :robots, rules: [
|
124
116
|
{
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
117
|
+
user_agent: '*',
|
118
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
119
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
128
120
|
}
|
129
121
|
]
|
130
122
|
"""
|
@@ -143,16 +135,16 @@ Feature: Middleman-Robots on build
|
|
143
135
|
Given a fixture app "basic-app"
|
144
136
|
And a file named "config.rb" with:
|
145
137
|
"""
|
146
|
-
activate :robots, :
|
138
|
+
activate :robots, rules: [
|
147
139
|
{
|
148
|
-
:
|
149
|
-
:
|
150
|
-
:
|
140
|
+
user_agent: 'Googlebot',
|
141
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
142
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
151
143
|
},
|
152
144
|
{
|
153
|
-
:
|
154
|
-
:
|
155
|
-
:
|
145
|
+
user_agent: 'Googlebot-Image',
|
146
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
147
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
156
148
|
}
|
157
149
|
]
|
158
150
|
"""
|
@@ -177,7 +169,7 @@ Feature: Middleman-Robots on build
|
|
177
169
|
Given a fixture app "basic-app"
|
178
170
|
And a file named "config.rb" with:
|
179
171
|
"""
|
180
|
-
activate :robots, :
|
172
|
+
activate :robots, sitemap: 'http://example.com/sitemap.xml'
|
181
173
|
"""
|
182
174
|
And a successfully built app at "basic-app"
|
183
175
|
When I cd to "build"
|
@@ -191,19 +183,19 @@ Feature: Middleman-Robots on build
|
|
191
183
|
And a file named "config.rb" with:
|
192
184
|
"""
|
193
185
|
activate :robots,
|
194
|
-
:
|
186
|
+
rules: [
|
195
187
|
{
|
196
|
-
:
|
197
|
-
:
|
198
|
-
:
|
188
|
+
user_agent:'Googlebot',
|
189
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
190
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
199
191
|
},
|
200
192
|
{
|
201
|
-
:
|
202
|
-
:
|
203
|
-
:
|
193
|
+
user_agent:'Googlebot-Image',
|
194
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
195
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
204
196
|
}
|
205
197
|
],
|
206
|
-
:
|
198
|
+
sitemap: 'http://example.com/sitemap.xml'
|
207
199
|
"""
|
208
200
|
And a successfully built app at "basic-app"
|
209
201
|
When I cd to "build"
|
data/features/support/env.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
1
|
+
rules = [
|
2
|
+
{
|
3
|
+
user_agent: 'Googlebot',
|
4
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
5
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
6
|
+
},
|
7
|
+
{
|
8
|
+
user_agent: 'Googlebot-Image',
|
9
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
10
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
11
|
+
}
|
12
|
+
]
|
13
|
+
|
14
|
+
activate :robots, rules: rules, sitemap: 'http://example.com/sitemap.xml'
|
data/lib/middleman-robots.rb
CHANGED
@@ -9,11 +9,7 @@ module Middleman
|
|
9
9
|
option :sitemap, false, 'URI of sitemap.xml'
|
10
10
|
|
11
11
|
def manipulate_resource_list(resources)
|
12
|
-
|
13
|
-
File.open(tmp_path, 'w+') do |f|
|
14
|
-
f.puts(Generator.new(options.rules, options.sitemap).process)
|
15
|
-
end
|
16
|
-
|
12
|
+
write_robots(options)
|
17
13
|
robots = Middleman::Sitemap::Resource.new(
|
18
14
|
app.sitemap,
|
19
15
|
'robots.txt',
|
@@ -23,6 +19,16 @@ module Middleman
|
|
23
19
|
logger.info '== middleman-robots: robots.txt added to resources =='
|
24
20
|
resources << robots
|
25
21
|
end
|
22
|
+
|
23
|
+
def write_robots(options)
|
24
|
+
File.open(tmp_path, 'w+') do |f|
|
25
|
+
f.puts(Generator.new(options.rules, options.sitemap).process)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def tmp_path
|
30
|
+
File.expand_path('../../../tmp/robots/robots.txt', __FILE__)
|
31
|
+
end
|
26
32
|
end
|
27
33
|
end
|
28
34
|
end
|
@@ -25,18 +25,19 @@ module Middleman
|
|
25
25
|
end
|
26
26
|
|
27
27
|
private
|
28
|
-
def block_text
|
29
|
-
return '' if @rules.empty?
|
30
|
-
data = []
|
31
|
-
@rules.each do |rule|
|
32
|
-
data << Group.new(rule).text
|
33
|
-
end
|
34
|
-
data.join("\n")
|
35
|
-
end
|
36
28
|
|
37
|
-
|
38
|
-
|
29
|
+
def block_text
|
30
|
+
return '' if @rules.empty?
|
31
|
+
data = []
|
32
|
+
@rules.each do |rule|
|
33
|
+
data << Group.new(rule).text
|
39
34
|
end
|
35
|
+
data.join("\n")
|
36
|
+
end
|
37
|
+
|
38
|
+
def sitemap_text
|
39
|
+
@sitemap_uri ? "Sitemap: #{@sitemap_uri}" : ''
|
40
|
+
end
|
40
41
|
end
|
41
42
|
end
|
42
43
|
end
|
@@ -15,36 +15,37 @@ module Middleman
|
|
15
15
|
def text
|
16
16
|
group = []
|
17
17
|
group << "User-Agent: #{@user_agent}" unless @user_agent.empty?
|
18
|
-
group << @disallow.collect{|item| "Disallow: #{item}" }.join("\n") if @disallow.length
|
19
|
-
group << @allow.collect{|item| "Allow: #{item}" }.join("\n") if @allow.length
|
18
|
+
group << @disallow.collect { |item| "Disallow: #{item}" }.join("\n") if @disallow.length.positive?
|
19
|
+
group << @allow.collect { |item| "Allow: #{item}" }.join("\n") if @allow.length.positive?
|
20
20
|
group.join("\n") + "\n"
|
21
21
|
end
|
22
22
|
|
23
23
|
private
|
24
|
-
def generate_user_agent(rule)
|
25
|
-
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
26
|
-
rule[:user_agent] || rule['user-agent']
|
27
|
-
end
|
28
24
|
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
def generate_user_agent(rule)
|
26
|
+
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
27
|
+
rule[:user_agent] || rule['user-agent']
|
28
|
+
end
|
29
|
+
|
30
|
+
def generate_disallow(rule)
|
31
|
+
paths = []
|
32
|
+
return paths unless rule.key?(:disallow)
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
paths
|
34
|
+
rule[:disallow].each do |path|
|
35
|
+
paths << File.join('/', path)
|
37
36
|
end
|
37
|
+
paths
|
38
|
+
end
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
def generate_allow(rule)
|
41
|
+
paths = []
|
42
|
+
return paths unless rule.key?(:allow)
|
42
43
|
|
43
|
-
|
44
|
-
|
45
|
-
end
|
46
|
-
paths
|
44
|
+
rule[:allow].each do |path|
|
45
|
+
paths << File.join('/', path)
|
47
46
|
end
|
47
|
+
paths
|
48
|
+
end
|
48
49
|
end
|
49
50
|
end
|
50
51
|
end
|
data/lib/middleman_extension.rb
CHANGED
@@ -1,2 +1 @@
|
|
1
|
-
require
|
2
|
-
|
1
|
+
require 'middleman-robots'
|
data/middleman-robots.gemspec
CHANGED
@@ -1,30 +1,29 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
lib = File.expand_path('../lib', __FILE__)
|
3
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
3
|
require 'middleman-robots/version'
|
5
4
|
|
6
5
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
6
|
+
spec.name = 'middleman-robots'
|
8
7
|
spec.version = Middleman::Robots::VERSION
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
14
|
-
spec.license =
|
8
|
+
spec.authors = ['Yuya Matsushima']
|
9
|
+
spec.email = ['terra@e2esound.com']
|
10
|
+
spec.summary = 'Generate robots.txt by config.rb.'
|
11
|
+
spec.description = 'middleman-robots create robots.txt includes Allow or Disallow and sitemap path.'
|
12
|
+
spec.homepage = 'https://github.com/yterajima/middleman-robots'
|
13
|
+
spec.license = 'MIT'
|
15
14
|
|
16
15
|
spec.files = `git ls-files -z`.split("\x0")
|
17
16
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
-
spec.require_paths = [
|
20
|
-
spec.required_ruby_version = '>= 2.
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
spec.required_ruby_version = '>= 2.2.0'
|
21
20
|
|
22
|
-
spec.add_runtime_dependency
|
21
|
+
spec.add_runtime_dependency 'middleman', '>= 4.0'
|
23
22
|
|
24
|
-
spec.add_development_dependency
|
25
|
-
spec.add_development_dependency
|
26
|
-
spec.add_development_dependency
|
27
|
-
spec.add_development_dependency
|
28
|
-
spec.add_development_dependency
|
23
|
+
spec.add_development_dependency 'aruba', '>= 0.14.3'
|
24
|
+
spec.add_development_dependency 'bundler', '>= 1.16'
|
25
|
+
spec.add_development_dependency 'capybara', '>= 2.18.0'
|
26
|
+
spec.add_development_dependency 'cucumber', '>= 3.1.0'
|
27
|
+
spec.add_development_dependency 'rake', '>= 12.3'
|
28
|
+
spec.add_development_dependency 'rubocop', '>= 0.52.1'
|
29
29
|
end
|
30
|
-
|
data/tests/test_generator.rb
CHANGED
@@ -1,24 +1,32 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
2
|
require_relative '../lib/middleman-robots/generator.rb'
|
3
3
|
|
4
|
-
class
|
4
|
+
class TestGenerator < MiniTest::Test
|
5
5
|
def test_process
|
6
|
-
|
6
|
+
sitemap_uri = 'http://example.com/sitemap.xml'
|
7
|
+
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
8
|
+
assert_equal expected, generator.process
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def rules
|
14
|
+
[
|
7
15
|
{
|
8
16
|
user_agent: 'Googlebot',
|
9
|
-
disallow: %w
|
10
|
-
allow: %w
|
17
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
18
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
11
19
|
},
|
12
20
|
{
|
13
21
|
user_agent: 'Googlebot-Image',
|
14
|
-
disallow: %w
|
15
|
-
allow: %w
|
22
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
23
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
16
24
|
}
|
17
25
|
]
|
18
|
-
|
19
|
-
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
26
|
+
end
|
20
27
|
|
21
|
-
|
28
|
+
def expected
|
29
|
+
"User-Agent: Googlebot
|
22
30
|
Disallow: /tmp/*
|
23
31
|
Disallow: /something/dir/file_disallow.html
|
24
32
|
Allow: /allow/*
|
@@ -31,7 +39,5 @@ Allow: /allow/*
|
|
31
39
|
Allow: /something/dir/file_allow.html
|
32
40
|
|
33
41
|
Sitemap: http://example.com/sitemap.xml"
|
34
|
-
|
35
|
-
assert_equal expected, generator.process
|
36
42
|
end
|
37
43
|
end
|
data/tests/test_group.rb
CHANGED
@@ -1,25 +1,24 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
2
|
require_relative '../lib/middleman-robots/group.rb'
|
3
3
|
|
4
|
-
class
|
5
|
-
|
4
|
+
class TestGroup < MiniTest::Test
|
6
5
|
def test_initialize
|
7
6
|
rule = {
|
8
7
|
user_agent: 'GoogleBot',
|
9
|
-
disallow: %w
|
10
|
-
allow: %w
|
8
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
9
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
11
10
|
}
|
12
11
|
group = Middleman::Robots::Group.new(rule)
|
13
12
|
|
14
13
|
assert_equal rule[:user_agent], group.user_agent
|
15
|
-
assert_equal %w
|
16
|
-
assert_equal %w
|
14
|
+
assert_equal %w[/tmp/* /someting/dir/disallow.html], group.disallow
|
15
|
+
assert_equal %w[/allow/* /someting/dir/allow.html], group.allow
|
17
16
|
end
|
18
17
|
|
19
18
|
def test_initialize_without_user_agent
|
20
19
|
rule = {
|
21
|
-
disallow: %w
|
22
|
-
allow: %w
|
20
|
+
disallow: %w[/tmp/*],
|
21
|
+
allow: %w[/allow/*]
|
23
22
|
}
|
24
23
|
group = Middleman::Robots::Group.new(rule)
|
25
24
|
|
@@ -31,7 +30,7 @@ class Test_Group < MiniTest::Test
|
|
31
30
|
def test_initialize_without_disallow
|
32
31
|
rule = {
|
33
32
|
user_agent: 'GoogleBot',
|
34
|
-
allow: %w
|
33
|
+
allow: %w[/allow/* /someting/dir/allow.html]
|
35
34
|
}
|
36
35
|
group = Middleman::Robots::Group.new(rule)
|
37
36
|
|
@@ -43,7 +42,7 @@ class Test_Group < MiniTest::Test
|
|
43
42
|
def test_initialize_without_allow
|
44
43
|
rule = {
|
45
44
|
user_agent: 'GoogleBot',
|
46
|
-
disallow: %w
|
45
|
+
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
47
46
|
}
|
48
47
|
group = Middleman::Robots::Group.new(rule)
|
49
48
|
|
@@ -55,34 +54,34 @@ class Test_Group < MiniTest::Test
|
|
55
54
|
def test_text
|
56
55
|
rule = {
|
57
56
|
user_agent: 'GoogleBot',
|
58
|
-
disallow: %w
|
59
|
-
allow: %w
|
57
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
58
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
60
59
|
}
|
61
60
|
group = Middleman::Robots::Group.new(rule)
|
62
61
|
|
63
|
-
expected =
|
64
|
-
User-Agent: GoogleBot
|
65
|
-
Disallow: /tmp/*
|
66
|
-
Disallow: /someting/dir/disallow.html
|
67
|
-
Allow: /allow/*
|
68
|
-
Allow: /someting/dir/allow.html
|
69
|
-
|
62
|
+
expected = <<~ROBOTS
|
63
|
+
User-Agent: GoogleBot
|
64
|
+
Disallow: /tmp/*
|
65
|
+
Disallow: /someting/dir/disallow.html
|
66
|
+
Allow: /allow/*
|
67
|
+
Allow: /someting/dir/allow.html
|
68
|
+
ROBOTS
|
70
69
|
|
71
70
|
assert_equal expected, group.text
|
72
71
|
end
|
73
72
|
|
74
73
|
def test_text_without_user_agent
|
75
74
|
rule = {
|
76
|
-
disallow: %w
|
77
|
-
allow: %w
|
75
|
+
disallow: %w[/tmp/*],
|
76
|
+
allow: %w[/allow/*]
|
78
77
|
}
|
79
78
|
group = Middleman::Robots::Group.new(rule)
|
80
79
|
|
81
|
-
expected =
|
82
|
-
User-Agent: *
|
83
|
-
Disallow: /tmp/*
|
84
|
-
Allow: /allow/*
|
85
|
-
|
80
|
+
expected = <<~ROBOTS
|
81
|
+
User-Agent: *
|
82
|
+
Disallow: /tmp/*
|
83
|
+
Allow: /allow/*
|
84
|
+
ROBOTS
|
86
85
|
|
87
86
|
assert_equal expected, group.text
|
88
87
|
end
|
@@ -90,15 +89,15 @@ Allow: /allow/*
|
|
90
89
|
def test_text_without_disallow
|
91
90
|
rule = {
|
92
91
|
user_agent: 'GoogleBot',
|
93
|
-
allow: %w
|
92
|
+
allow: %w[/allow/* /someting/dir/allow.html]
|
94
93
|
}
|
95
94
|
group = Middleman::Robots::Group.new(rule)
|
96
95
|
|
97
|
-
expected =
|
98
|
-
User-Agent: GoogleBot
|
99
|
-
Allow: /allow/*
|
100
|
-
Allow: /someting/dir/allow.html
|
101
|
-
|
96
|
+
expected = <<~ROBOTS
|
97
|
+
User-Agent: GoogleBot
|
98
|
+
Allow: /allow/*
|
99
|
+
Allow: /someting/dir/allow.html
|
100
|
+
ROBOTS
|
102
101
|
|
103
102
|
assert_equal expected, group.text
|
104
103
|
end
|
@@ -106,15 +105,15 @@ Allow: /someting/dir/allow.html
|
|
106
105
|
def test_text_without_allow
|
107
106
|
rule = {
|
108
107
|
user_agent: 'GoogleBot',
|
109
|
-
disallow: %w
|
108
|
+
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
110
109
|
}
|
111
110
|
group = Middleman::Robots::Group.new(rule)
|
112
111
|
|
113
|
-
expected =
|
114
|
-
User-Agent: GoogleBot
|
115
|
-
Disallow: /tmp/*
|
116
|
-
Disallow: /someting/dir/disallow.html
|
117
|
-
|
112
|
+
expected = <<~ROBOTS
|
113
|
+
User-Agent: GoogleBot
|
114
|
+
Disallow: /tmp/*
|
115
|
+
Disallow: /someting/dir/disallow.html
|
116
|
+
ROBOTS
|
118
117
|
|
119
118
|
assert_equal expected, group.text
|
120
119
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: middleman-robots
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuya Matsushima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: middleman
|
@@ -25,76 +25,91 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: aruba
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.14.3
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.14.3
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '1.16'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '1.16'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: capybara
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 2.18.0
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 2.18.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: cucumber
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 3.1.0
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: 3.1.0
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '12.3'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '12.3'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.52.1
|
90
104
|
type: :development
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
97
|
-
description:
|
110
|
+
version: 0.52.1
|
111
|
+
description: middleman-robots create robots.txt includes Allow or Disallow and sitemap
|
112
|
+
path.
|
98
113
|
email:
|
99
114
|
- terra@e2esound.com
|
100
115
|
executables: []
|
@@ -102,6 +117,7 @@ extensions: []
|
|
102
117
|
extra_rdoc_files: []
|
103
118
|
files:
|
104
119
|
- ".gitignore"
|
120
|
+
- ".rubocop.yml"
|
105
121
|
- ".travis.yml"
|
106
122
|
- Gemfile
|
107
123
|
- LICENSE.txt
|
@@ -138,7 +154,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
138
154
|
requirements:
|
139
155
|
- - ">="
|
140
156
|
- !ruby/object:Gem::Version
|
141
|
-
version: 2.
|
157
|
+
version: 2.2.0
|
142
158
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
143
159
|
requirements:
|
144
160
|
- - ">="
|
@@ -146,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
162
|
version: '0'
|
147
163
|
requirements: []
|
148
164
|
rubyforge_project:
|
149
|
-
rubygems_version: 2.
|
165
|
+
rubygems_version: 2.7.3
|
150
166
|
signing_key:
|
151
167
|
specification_version: 4
|
152
168
|
summary: Generate robots.txt by config.rb.
|