middleman-robots 1.3.2 → 1.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.rubocop.yml +38 -0
- data/.travis.yml +7 -5
- data/README.md +15 -15
- data/features/build.feature +30 -38
- data/features/support/env.rb +0 -1
- data/fixtures/server-app/config.rb +14 -14
- data/lib/middleman-robots.rb +2 -2
- data/lib/middleman-robots/extension.rb +11 -5
- data/lib/middleman-robots/generator.rb +11 -10
- data/lib/middleman-robots/group.rb +21 -20
- data/lib/middleman-robots/version.rb +1 -1
- data/lib/middleman_extension.rb +1 -2
- data/middleman-robots.gemspec +16 -17
- data/tests/test_generator.rb +17 -11
- data/tests/test_group.rb +37 -38
- metadata +37 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: b1f3a8305157e44491560743f0aff0ea58b50bf46f5eb10c61482cea721d064b
|
4
|
+
data.tar.gz: 5ffa930584f52dc32fdbbe9ed87edc58566ab560ca5357499d2323409dbe6aa2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb58e902248a63782ddd07779b963a0b9f0baa742c3bb109386577c7942be9efb23f63f201b60aecebd0cf02fea6340d7ba5305a84607f47bbef1b2d34c9d04d
|
7
|
+
data.tar.gz: '08205a13c77a299db354736153d059b40bc27ce4167b719d93bda74a3580cfe881ec39da91d2adab6d9f30a7fa88765e591584c8c8fc695713541b0059cfc7c8'
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
AllCops:
|
2
|
+
Exclude:
|
3
|
+
- 'pkg/*'
|
4
|
+
- 'tmp/**/*'
|
5
|
+
- 'vendor/**/*'
|
6
|
+
TargetRubyVersion: 2.5
|
7
|
+
DisplayCopNames: true
|
8
|
+
|
9
|
+
Rails:
|
10
|
+
Enabled: false
|
11
|
+
|
12
|
+
Style/ClassAndModuleChildren:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
Style/Documentation:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Style/FrozenStringLiteralComment:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/RedundantSelf:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Gemspec/RequiredRubyVersion:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Metrics/LineLength:
|
28
|
+
Max: 120
|
29
|
+
Exclude:
|
30
|
+
- '*.gemspec'
|
31
|
+
|
32
|
+
Metrics/MethodLength:
|
33
|
+
Max: 20
|
34
|
+
|
35
|
+
Naming/FileName:
|
36
|
+
Enabled: true
|
37
|
+
Exclude:
|
38
|
+
- 'lib/middleman-robots.rb'
|
data/.travis.yml
CHANGED
@@ -5,16 +5,18 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.
|
9
|
-
- 2.3
|
10
|
-
- 2.
|
8
|
+
- 2.5.0
|
9
|
+
- 2.4.3
|
10
|
+
- 2.3.6
|
11
11
|
os:
|
12
12
|
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
|
-
- rvm:
|
17
|
-
script:
|
16
|
+
- rvm: ruby-head
|
17
|
+
script:
|
18
|
+
- bundle exec rake test
|
19
|
+
- bundle exec rubocop
|
18
20
|
cache: bundler
|
19
21
|
env: TEST=true
|
20
22
|
notifications:
|
data/README.md
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/middleman-robots.svg)](http://badge.fury.io/rb/middleman-robots)
|
4
4
|
[![Build Status](https://travis-ci.org/yterajima/middleman-robots.svg?branch=master)](https://travis-ci.org/yterajima/middleman-robots)
|
5
5
|
|
6
|
-
`middleman-robots` is an extension of [Middleman](
|
6
|
+
`middleman-robots` is an extension of [Middleman](https://middlemanapp.com/). This can create `robots.txt`.
|
7
7
|
|
8
8
|
This plugin support Middleman v3-stable and v4.
|
9
9
|
|
@@ -26,10 +26,10 @@ Or install it yourself as:
|
|
26
26
|
```ruby
|
27
27
|
# config.rb
|
28
28
|
activate :robots,
|
29
|
-
:
|
30
|
-
{:
|
29
|
+
rules: [
|
30
|
+
{ user_agent: '*', allow: %w[/] }
|
31
31
|
],
|
32
|
-
:
|
32
|
+
sitemap: 'https://example.com/sitemap.xml'
|
33
33
|
```
|
34
34
|
|
35
35
|
Created `robots.txt`:
|
@@ -38,28 +38,28 @@ Created `robots.txt`:
|
|
38
38
|
User-Agent: *
|
39
39
|
Allow: /
|
40
40
|
|
41
|
-
Sitemap:
|
41
|
+
Sitemap: https://example.com/sitemap.xml
|
42
42
|
|
43
43
|
```
|
44
44
|
|
45
|
-
You can use options,
|
45
|
+
You can use options, `rules` [ {`user_agent`(string), `allow`(array), `disallow`(array)}] and `sitemap`. Like this:
|
46
46
|
|
47
47
|
```ruby
|
48
48
|
# config.rb
|
49
49
|
activate :robots,
|
50
|
-
:
|
50
|
+
rules: [
|
51
51
|
{
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
52
|
+
user_agent: 'Googlebot',
|
53
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
54
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
55
55
|
},
|
56
56
|
{
|
57
|
-
:
|
58
|
-
:
|
59
|
-
:
|
57
|
+
user_agent: 'Googlebot-Image',
|
58
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
59
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
60
60
|
}
|
61
61
|
],
|
62
|
-
:
|
62
|
+
sitemap: 'https://example.com/sitemap.xml'
|
63
63
|
```
|
64
64
|
|
65
65
|
Created `robots.txt`:
|
@@ -77,7 +77,7 @@ Disallow: /something/dir/file_disallow.html
|
|
77
77
|
Allow: /allow/
|
78
78
|
Allow: /something/dir/file_allow.html
|
79
79
|
|
80
|
-
Sitemap:
|
80
|
+
Sitemap: https://example.com/sitemap.xml
|
81
81
|
|
82
82
|
```
|
83
83
|
|
data/features/build.feature
CHANGED
@@ -18,9 +18,7 @@ Feature: Middleman-Robots on build
|
|
18
18
|
Given a fixture app "basic-app"
|
19
19
|
And a file named "config.rb" with:
|
20
20
|
"""
|
21
|
-
activate :robots, :
|
22
|
-
{:user_agent => '*'}
|
23
|
-
]
|
21
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
24
22
|
"""
|
25
23
|
And a successfully built app at "basic-app"
|
26
24
|
When I cd to "build"
|
@@ -34,9 +32,7 @@ Feature: Middleman-Robots on build
|
|
34
32
|
And a file named "config.rb" with:
|
35
33
|
"""
|
36
34
|
activate :robots do |r|
|
37
|
-
r.rules = [
|
38
|
-
{:user_agent => '*'}
|
39
|
-
]
|
35
|
+
r.rules = [ { user_agent: '*'} ]
|
40
36
|
end
|
41
37
|
"""
|
42
38
|
And a successfully built app at "basic-app"
|
@@ -50,9 +46,7 @@ Feature: Middleman-Robots on build
|
|
50
46
|
Given a fixture app "basic-app"
|
51
47
|
And a file named "config.rb" with:
|
52
48
|
"""
|
53
|
-
activate :robots, :
|
54
|
-
{:user_agent => '*'}
|
55
|
-
]
|
49
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
56
50
|
"""
|
57
51
|
And a successfully built app at "basic-app"
|
58
52
|
When I cd to "build"
|
@@ -65,9 +59,7 @@ Feature: Middleman-Robots on build
|
|
65
59
|
Given a fixture app "basic-app"
|
66
60
|
And a file named "config.rb" with:
|
67
61
|
"""
|
68
|
-
activate :robots, :
|
69
|
-
{'user-agent' => '*'}
|
70
|
-
]
|
62
|
+
activate :robots, rules: [ { 'user-agent': '*' } ]
|
71
63
|
"""
|
72
64
|
And a successfully built app at "basic-app"
|
73
65
|
When I cd to "build"
|
@@ -80,10 +72,10 @@ Feature: Middleman-Robots on build
|
|
80
72
|
Given a fixture app "basic-app"
|
81
73
|
And a file named "config.rb" with:
|
82
74
|
"""
|
83
|
-
activate :robots, :
|
75
|
+
activate :robots, rules: [
|
84
76
|
{
|
85
|
-
:
|
86
|
-
:
|
77
|
+
user_agent: '*',
|
78
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html]
|
87
79
|
}
|
88
80
|
]
|
89
81
|
"""
|
@@ -100,10 +92,10 @@ Feature: Middleman-Robots on build
|
|
100
92
|
Given a fixture app "basic-app"
|
101
93
|
And a file named "config.rb" with:
|
102
94
|
"""
|
103
|
-
activate :robots, :
|
95
|
+
activate :robots, rules: [
|
104
96
|
{
|
105
|
-
:
|
106
|
-
:
|
97
|
+
user_agent: '*',
|
98
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
107
99
|
}
|
108
100
|
]
|
109
101
|
"""
|
@@ -120,11 +112,11 @@ Feature: Middleman-Robots on build
|
|
120
112
|
Given a fixture app "basic-app"
|
121
113
|
And a file named "config.rb" with:
|
122
114
|
"""
|
123
|
-
activate :robots, :
|
115
|
+
activate :robots, rules: [
|
124
116
|
{
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
117
|
+
user_agent: '*',
|
118
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
119
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
128
120
|
}
|
129
121
|
]
|
130
122
|
"""
|
@@ -143,16 +135,16 @@ Feature: Middleman-Robots on build
|
|
143
135
|
Given a fixture app "basic-app"
|
144
136
|
And a file named "config.rb" with:
|
145
137
|
"""
|
146
|
-
activate :robots, :
|
138
|
+
activate :robots, rules: [
|
147
139
|
{
|
148
|
-
:
|
149
|
-
:
|
150
|
-
:
|
140
|
+
user_agent: 'Googlebot',
|
141
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
142
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
151
143
|
},
|
152
144
|
{
|
153
|
-
:
|
154
|
-
:
|
155
|
-
:
|
145
|
+
user_agent: 'Googlebot-Image',
|
146
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
147
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
156
148
|
}
|
157
149
|
]
|
158
150
|
"""
|
@@ -177,7 +169,7 @@ Feature: Middleman-Robots on build
|
|
177
169
|
Given a fixture app "basic-app"
|
178
170
|
And a file named "config.rb" with:
|
179
171
|
"""
|
180
|
-
activate :robots, :
|
172
|
+
activate :robots, sitemap: 'http://example.com/sitemap.xml'
|
181
173
|
"""
|
182
174
|
And a successfully built app at "basic-app"
|
183
175
|
When I cd to "build"
|
@@ -191,19 +183,19 @@ Feature: Middleman-Robots on build
|
|
191
183
|
And a file named "config.rb" with:
|
192
184
|
"""
|
193
185
|
activate :robots,
|
194
|
-
:
|
186
|
+
rules: [
|
195
187
|
{
|
196
|
-
:
|
197
|
-
:
|
198
|
-
:
|
188
|
+
user_agent:'Googlebot',
|
189
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
190
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
199
191
|
},
|
200
192
|
{
|
201
|
-
:
|
202
|
-
:
|
203
|
-
:
|
193
|
+
user_agent:'Googlebot-Image',
|
194
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
195
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
204
196
|
}
|
205
197
|
],
|
206
|
-
:
|
198
|
+
sitemap: 'http://example.com/sitemap.xml'
|
207
199
|
"""
|
208
200
|
And a successfully built app at "basic-app"
|
209
201
|
When I cd to "build"
|
data/features/support/env.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
1
|
+
rules = [
|
2
|
+
{
|
3
|
+
user_agent: 'Googlebot',
|
4
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
5
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
6
|
+
},
|
7
|
+
{
|
8
|
+
user_agent: 'Googlebot-Image',
|
9
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
10
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
11
|
+
}
|
12
|
+
]
|
13
|
+
|
14
|
+
activate :robots, rules: rules, sitemap: 'http://example.com/sitemap.xml'
|
data/lib/middleman-robots.rb
CHANGED
@@ -9,11 +9,7 @@ module Middleman
|
|
9
9
|
option :sitemap, false, 'URI of sitemap.xml'
|
10
10
|
|
11
11
|
def manipulate_resource_list(resources)
|
12
|
-
|
13
|
-
File.open(tmp_path, 'w+') do |f|
|
14
|
-
f.puts(Generator.new(options.rules, options.sitemap).process)
|
15
|
-
end
|
16
|
-
|
12
|
+
write_robots(options)
|
17
13
|
robots = Middleman::Sitemap::Resource.new(
|
18
14
|
app.sitemap,
|
19
15
|
'robots.txt',
|
@@ -23,6 +19,16 @@ module Middleman
|
|
23
19
|
logger.info '== middleman-robots: robots.txt added to resources =='
|
24
20
|
resources << robots
|
25
21
|
end
|
22
|
+
|
23
|
+
def write_robots(options)
|
24
|
+
File.open(tmp_path, 'w+') do |f|
|
25
|
+
f.puts(Generator.new(options.rules, options.sitemap).process)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def tmp_path
|
30
|
+
File.expand_path('../../../tmp/robots/robots.txt', __FILE__)
|
31
|
+
end
|
26
32
|
end
|
27
33
|
end
|
28
34
|
end
|
@@ -25,18 +25,19 @@ module Middleman
|
|
25
25
|
end
|
26
26
|
|
27
27
|
private
|
28
|
-
def block_text
|
29
|
-
return '' if @rules.empty?
|
30
|
-
data = []
|
31
|
-
@rules.each do |rule|
|
32
|
-
data << Group.new(rule).text
|
33
|
-
end
|
34
|
-
data.join("\n")
|
35
|
-
end
|
36
28
|
|
37
|
-
|
38
|
-
|
29
|
+
def block_text
|
30
|
+
return '' if @rules.empty?
|
31
|
+
data = []
|
32
|
+
@rules.each do |rule|
|
33
|
+
data << Group.new(rule).text
|
39
34
|
end
|
35
|
+
data.join("\n")
|
36
|
+
end
|
37
|
+
|
38
|
+
def sitemap_text
|
39
|
+
@sitemap_uri ? "Sitemap: #{@sitemap_uri}" : ''
|
40
|
+
end
|
40
41
|
end
|
41
42
|
end
|
42
43
|
end
|
@@ -15,36 +15,37 @@ module Middleman
|
|
15
15
|
def text
|
16
16
|
group = []
|
17
17
|
group << "User-Agent: #{@user_agent}" unless @user_agent.empty?
|
18
|
-
group << @disallow.collect{|item| "Disallow: #{item}" }.join("\n") if @disallow.length
|
19
|
-
group << @allow.collect{|item| "Allow: #{item}" }.join("\n") if @allow.length
|
18
|
+
group << @disallow.collect { |item| "Disallow: #{item}" }.join("\n") if @disallow.length.positive?
|
19
|
+
group << @allow.collect { |item| "Allow: #{item}" }.join("\n") if @allow.length.positive?
|
20
20
|
group.join("\n") + "\n"
|
21
21
|
end
|
22
22
|
|
23
23
|
private
|
24
|
-
def generate_user_agent(rule)
|
25
|
-
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
26
|
-
rule[:user_agent] || rule['user-agent']
|
27
|
-
end
|
28
24
|
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
def generate_user_agent(rule)
|
26
|
+
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
27
|
+
rule[:user_agent] || rule['user-agent']
|
28
|
+
end
|
29
|
+
|
30
|
+
def generate_disallow(rule)
|
31
|
+
paths = []
|
32
|
+
return paths unless rule.key?(:disallow)
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
paths
|
34
|
+
rule[:disallow].each do |path|
|
35
|
+
paths << File.join('/', path)
|
37
36
|
end
|
37
|
+
paths
|
38
|
+
end
|
38
39
|
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
def generate_allow(rule)
|
41
|
+
paths = []
|
42
|
+
return paths unless rule.key?(:allow)
|
42
43
|
|
43
|
-
|
44
|
-
|
45
|
-
end
|
46
|
-
paths
|
44
|
+
rule[:allow].each do |path|
|
45
|
+
paths << File.join('/', path)
|
47
46
|
end
|
47
|
+
paths
|
48
|
+
end
|
48
49
|
end
|
49
50
|
end
|
50
51
|
end
|
data/lib/middleman_extension.rb
CHANGED
@@ -1,2 +1 @@
|
|
1
|
-
require
|
2
|
-
|
1
|
+
require 'middleman-robots'
|
data/middleman-robots.gemspec
CHANGED
@@ -1,30 +1,29 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
lib = File.expand_path('../lib', __FILE__)
|
3
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
3
|
require 'middleman-robots/version'
|
5
4
|
|
6
5
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
6
|
+
spec.name = 'middleman-robots'
|
8
7
|
spec.version = Middleman::Robots::VERSION
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
14
|
-
spec.license =
|
8
|
+
spec.authors = ['Yuya Matsushima']
|
9
|
+
spec.email = ['terra@e2esound.com']
|
10
|
+
spec.summary = 'Generate robots.txt by config.rb.'
|
11
|
+
spec.description = 'middleman-robots create robots.txt includes Allow or Disallow and sitemap path.'
|
12
|
+
spec.homepage = 'https://github.com/yterajima/middleman-robots'
|
13
|
+
spec.license = 'MIT'
|
15
14
|
|
16
15
|
spec.files = `git ls-files -z`.split("\x0")
|
17
16
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
-
spec.require_paths = [
|
20
|
-
spec.required_ruby_version = '>= 2.
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
spec.required_ruby_version = '>= 2.2.0'
|
21
20
|
|
22
|
-
spec.add_runtime_dependency
|
21
|
+
spec.add_runtime_dependency 'middleman', '>= 4.0'
|
23
22
|
|
24
|
-
spec.add_development_dependency
|
25
|
-
spec.add_development_dependency
|
26
|
-
spec.add_development_dependency
|
27
|
-
spec.add_development_dependency
|
28
|
-
spec.add_development_dependency
|
23
|
+
spec.add_development_dependency 'aruba', '>= 0.14.3'
|
24
|
+
spec.add_development_dependency 'bundler', '>= 1.16'
|
25
|
+
spec.add_development_dependency 'capybara', '>= 2.18.0'
|
26
|
+
spec.add_development_dependency 'cucumber', '>= 3.1.0'
|
27
|
+
spec.add_development_dependency 'rake', '>= 12.3'
|
28
|
+
spec.add_development_dependency 'rubocop', '>= 0.52.1'
|
29
29
|
end
|
30
|
-
|
data/tests/test_generator.rb
CHANGED
@@ -1,24 +1,32 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
2
|
require_relative '../lib/middleman-robots/generator.rb'
|
3
3
|
|
4
|
-
class
|
4
|
+
class TestGenerator < MiniTest::Test
|
5
5
|
def test_process
|
6
|
-
|
6
|
+
sitemap_uri = 'http://example.com/sitemap.xml'
|
7
|
+
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
8
|
+
assert_equal expected, generator.process
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def rules
|
14
|
+
[
|
7
15
|
{
|
8
16
|
user_agent: 'Googlebot',
|
9
|
-
disallow: %w
|
10
|
-
allow: %w
|
17
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
18
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
11
19
|
},
|
12
20
|
{
|
13
21
|
user_agent: 'Googlebot-Image',
|
14
|
-
disallow: %w
|
15
|
-
allow: %w
|
22
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
23
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
16
24
|
}
|
17
25
|
]
|
18
|
-
|
19
|
-
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
26
|
+
end
|
20
27
|
|
21
|
-
|
28
|
+
def expected
|
29
|
+
"User-Agent: Googlebot
|
22
30
|
Disallow: /tmp/*
|
23
31
|
Disallow: /something/dir/file_disallow.html
|
24
32
|
Allow: /allow/*
|
@@ -31,7 +39,5 @@ Allow: /allow/*
|
|
31
39
|
Allow: /something/dir/file_allow.html
|
32
40
|
|
33
41
|
Sitemap: http://example.com/sitemap.xml"
|
34
|
-
|
35
|
-
assert_equal expected, generator.process
|
36
42
|
end
|
37
43
|
end
|
data/tests/test_group.rb
CHANGED
@@ -1,25 +1,24 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
2
|
require_relative '../lib/middleman-robots/group.rb'
|
3
3
|
|
4
|
-
class
|
5
|
-
|
4
|
+
class TestGroup < MiniTest::Test
|
6
5
|
def test_initialize
|
7
6
|
rule = {
|
8
7
|
user_agent: 'GoogleBot',
|
9
|
-
disallow: %w
|
10
|
-
allow: %w
|
8
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
9
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
11
10
|
}
|
12
11
|
group = Middleman::Robots::Group.new(rule)
|
13
12
|
|
14
13
|
assert_equal rule[:user_agent], group.user_agent
|
15
|
-
assert_equal %w
|
16
|
-
assert_equal %w
|
14
|
+
assert_equal %w[/tmp/* /someting/dir/disallow.html], group.disallow
|
15
|
+
assert_equal %w[/allow/* /someting/dir/allow.html], group.allow
|
17
16
|
end
|
18
17
|
|
19
18
|
def test_initialize_without_user_agent
|
20
19
|
rule = {
|
21
|
-
disallow: %w
|
22
|
-
allow: %w
|
20
|
+
disallow: %w[/tmp/*],
|
21
|
+
allow: %w[/allow/*]
|
23
22
|
}
|
24
23
|
group = Middleman::Robots::Group.new(rule)
|
25
24
|
|
@@ -31,7 +30,7 @@ class Test_Group < MiniTest::Test
|
|
31
30
|
def test_initialize_without_disallow
|
32
31
|
rule = {
|
33
32
|
user_agent: 'GoogleBot',
|
34
|
-
allow: %w
|
33
|
+
allow: %w[/allow/* /someting/dir/allow.html]
|
35
34
|
}
|
36
35
|
group = Middleman::Robots::Group.new(rule)
|
37
36
|
|
@@ -43,7 +42,7 @@ class Test_Group < MiniTest::Test
|
|
43
42
|
def test_initialize_without_allow
|
44
43
|
rule = {
|
45
44
|
user_agent: 'GoogleBot',
|
46
|
-
disallow: %w
|
45
|
+
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
47
46
|
}
|
48
47
|
group = Middleman::Robots::Group.new(rule)
|
49
48
|
|
@@ -55,34 +54,34 @@ class Test_Group < MiniTest::Test
|
|
55
54
|
def test_text
|
56
55
|
rule = {
|
57
56
|
user_agent: 'GoogleBot',
|
58
|
-
disallow: %w
|
59
|
-
allow: %w
|
57
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
58
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
60
59
|
}
|
61
60
|
group = Middleman::Robots::Group.new(rule)
|
62
61
|
|
63
|
-
expected =
|
64
|
-
User-Agent: GoogleBot
|
65
|
-
Disallow: /tmp/*
|
66
|
-
Disallow: /someting/dir/disallow.html
|
67
|
-
Allow: /allow/*
|
68
|
-
Allow: /someting/dir/allow.html
|
69
|
-
|
62
|
+
expected = <<~ROBOTS
|
63
|
+
User-Agent: GoogleBot
|
64
|
+
Disallow: /tmp/*
|
65
|
+
Disallow: /someting/dir/disallow.html
|
66
|
+
Allow: /allow/*
|
67
|
+
Allow: /someting/dir/allow.html
|
68
|
+
ROBOTS
|
70
69
|
|
71
70
|
assert_equal expected, group.text
|
72
71
|
end
|
73
72
|
|
74
73
|
def test_text_without_user_agent
|
75
74
|
rule = {
|
76
|
-
disallow: %w
|
77
|
-
allow: %w
|
75
|
+
disallow: %w[/tmp/*],
|
76
|
+
allow: %w[/allow/*]
|
78
77
|
}
|
79
78
|
group = Middleman::Robots::Group.new(rule)
|
80
79
|
|
81
|
-
expected =
|
82
|
-
User-Agent: *
|
83
|
-
Disallow: /tmp/*
|
84
|
-
Allow: /allow/*
|
85
|
-
|
80
|
+
expected = <<~ROBOTS
|
81
|
+
User-Agent: *
|
82
|
+
Disallow: /tmp/*
|
83
|
+
Allow: /allow/*
|
84
|
+
ROBOTS
|
86
85
|
|
87
86
|
assert_equal expected, group.text
|
88
87
|
end
|
@@ -90,15 +89,15 @@ Allow: /allow/*
|
|
90
89
|
def test_text_without_disallow
|
91
90
|
rule = {
|
92
91
|
user_agent: 'GoogleBot',
|
93
|
-
allow: %w
|
92
|
+
allow: %w[/allow/* /someting/dir/allow.html]
|
94
93
|
}
|
95
94
|
group = Middleman::Robots::Group.new(rule)
|
96
95
|
|
97
|
-
expected =
|
98
|
-
User-Agent: GoogleBot
|
99
|
-
Allow: /allow/*
|
100
|
-
Allow: /someting/dir/allow.html
|
101
|
-
|
96
|
+
expected = <<~ROBOTS
|
97
|
+
User-Agent: GoogleBot
|
98
|
+
Allow: /allow/*
|
99
|
+
Allow: /someting/dir/allow.html
|
100
|
+
ROBOTS
|
102
101
|
|
103
102
|
assert_equal expected, group.text
|
104
103
|
end
|
@@ -106,15 +105,15 @@ Allow: /someting/dir/allow.html
|
|
106
105
|
def test_text_without_allow
|
107
106
|
rule = {
|
108
107
|
user_agent: 'GoogleBot',
|
109
|
-
disallow: %w
|
108
|
+
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
110
109
|
}
|
111
110
|
group = Middleman::Robots::Group.new(rule)
|
112
111
|
|
113
|
-
expected =
|
114
|
-
User-Agent: GoogleBot
|
115
|
-
Disallow: /tmp/*
|
116
|
-
Disallow: /someting/dir/disallow.html
|
117
|
-
|
112
|
+
expected = <<~ROBOTS
|
113
|
+
User-Agent: GoogleBot
|
114
|
+
Disallow: /tmp/*
|
115
|
+
Disallow: /someting/dir/disallow.html
|
116
|
+
ROBOTS
|
118
117
|
|
119
118
|
assert_equal expected, group.text
|
120
119
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: middleman-robots
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuya Matsushima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: middleman
|
@@ -25,76 +25,91 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: aruba
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.14.3
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.14.3
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '1.16'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '1.16'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: capybara
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 2.18.0
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 2.18.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: cucumber
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 3.1.0
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: 3.1.0
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '12.3'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '12.3'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.52.1
|
90
104
|
type: :development
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
97
|
-
description:
|
110
|
+
version: 0.52.1
|
111
|
+
description: middleman-robots create robots.txt includes Allow or Disallow and sitemap
|
112
|
+
path.
|
98
113
|
email:
|
99
114
|
- terra@e2esound.com
|
100
115
|
executables: []
|
@@ -102,6 +117,7 @@ extensions: []
|
|
102
117
|
extra_rdoc_files: []
|
103
118
|
files:
|
104
119
|
- ".gitignore"
|
120
|
+
- ".rubocop.yml"
|
105
121
|
- ".travis.yml"
|
106
122
|
- Gemfile
|
107
123
|
- LICENSE.txt
|
@@ -138,7 +154,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
138
154
|
requirements:
|
139
155
|
- - ">="
|
140
156
|
- !ruby/object:Gem::Version
|
141
|
-
version: 2.
|
157
|
+
version: 2.2.0
|
142
158
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
143
159
|
requirements:
|
144
160
|
- - ">="
|
@@ -146,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
162
|
version: '0'
|
147
163
|
requirements: []
|
148
164
|
rubyforge_project:
|
149
|
-
rubygems_version: 2.
|
165
|
+
rubygems_version: 2.7.3
|
150
166
|
signing_key:
|
151
167
|
specification_version: 4
|
152
168
|
summary: Generate robots.txt by config.rb.
|