middleman-robots 1.3.2 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +25 -0
- data/.rspec +2 -0
- data/.rubocop.yml +28 -0
- data/.travis.yml +8 -9
- data/Gemfile +2 -0
- data/README.md +16 -16
- data/Rakefile +3 -3
- data/cucumber.yml +1 -0
- data/features/build.feature +30 -39
- data/features/support/env.rb +3 -2
- data/fixtures/server-app/config.rb +28 -14
- data/lib/middleman-robots.rb +4 -2
- data/lib/middleman-robots/extension.rb +13 -8
- data/lib/middleman-robots/generator.rb +12 -26
- data/lib/middleman-robots/generators/block.rb +55 -0
- data/lib/middleman-robots/generators/blocks.rb +26 -0
- data/lib/middleman-robots/generators/sitemap_uri.rb +24 -0
- data/lib/middleman-robots/version.rb +3 -1
- data/lib/middleman_extension.rb +2 -1
- data/middleman-robots.gemspec +21 -18
- data/spec/lib/middleman-robots/generator_spec.rb +65 -0
- data/spec/lib/middleman-robots/generators/block_spec.rb +205 -0
- data/spec/lib/middleman-robots/generators/blocks_spec.rb +50 -0
- data/spec/lib/middleman-robots/generators/sitemap_uri_spec.rb +51 -0
- data/spec/spec_helper.rb +13 -0
- metadata +97 -29
- data/lib/middleman-robots/group.rb +0 -50
- data/tests/test_generator.rb +0 -37
- data/tests/test_group.rb +0 -121
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 525009b32d148d32e07a4373e7532a46126e2b271ffef6221ad4d750d5a9b4a9
|
4
|
+
data.tar.gz: 45366b77e1e3ca26d889d4cead667c3826387063c48a5ae97dd12cde7f470d79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a9826c1a46025521f1fb884a8c17581e9d3aa9b3226fe2e37487ba1b33fd70c978c52a9b0d05164ee288e4d3d2c30597671c4c0a6809be94c72cea5f9821c31b
|
7
|
+
data.tar.gz: f152418cb899e8874ef4fbe52a79b785bb3cb9c3b7d08adb37c444f035f2e8c87319a977aa0e095b0b4683d909da6d9031fc1a46835c4c9601f49930634089d3
|
@@ -0,0 +1,25 @@
|
|
1
|
+
on: [push]
|
2
|
+
name: CI
|
3
|
+
jobs:
|
4
|
+
test:
|
5
|
+
strategy:
|
6
|
+
matrix:
|
7
|
+
ruby-version: [2.6, 2.7, 3.0]
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
steps:
|
10
|
+
- uses: actions/checkout@v2
|
11
|
+
- uses: ruby/setup-ruby@v1
|
12
|
+
with:
|
13
|
+
ruby-version: ${{ matrix.ruby-version }}
|
14
|
+
bundler-cache: true
|
15
|
+
- run: bundle exec rake test
|
16
|
+
|
17
|
+
rubocop:
|
18
|
+
runs-on: ubuntu-latest
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
- uses: ruby/setup-ruby@v1
|
22
|
+
with:
|
23
|
+
ruby-version: 3.0
|
24
|
+
bundler-cache: true
|
25
|
+
- run: bundle exec rubocop
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
AllCops:
|
2
|
+
NewCops: enable
|
3
|
+
Exclude:
|
4
|
+
- 'pkg/*'
|
5
|
+
- 'tmp/**/*'
|
6
|
+
- 'vendor/**/*'
|
7
|
+
TargetRubyVersion: 2.6
|
8
|
+
DisplayCopNames: true
|
9
|
+
|
10
|
+
Style/ClassAndModuleChildren:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Style/Documentation:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Metrics/BlockLength:
|
17
|
+
Exclude:
|
18
|
+
- 'spec/**/*_spec.rb'
|
19
|
+
|
20
|
+
Layout/LineLength:
|
21
|
+
Max: 100
|
22
|
+
Exclude:
|
23
|
+
- '*.gemspec'
|
24
|
+
|
25
|
+
Naming/FileName:
|
26
|
+
Enabled: true
|
27
|
+
Exclude:
|
28
|
+
- 'lib/middleman-robots.rb'
|
data/.travis.yml
CHANGED
@@ -5,18 +5,17 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.4
|
9
|
-
- 2.
|
10
|
-
- 2.
|
8
|
+
- 2.6.4
|
9
|
+
- 2.5.6
|
10
|
+
- 2.4.7
|
11
11
|
os:
|
12
|
-
- linux
|
12
|
+
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
|
-
- rvm:
|
17
|
-
script:
|
16
|
+
- rvm: ruby-head
|
17
|
+
script:
|
18
|
+
- bundle exec rubocop
|
19
|
+
- bundle exec rake test
|
18
20
|
cache: bundler
|
19
21
|
env: TEST=true
|
20
|
-
notifications:
|
21
|
-
slack:
|
22
|
-
secure: C8xb94J4fstzAI7t9GcvKQieKc1k+ECA96ydcxDwBI1fAbupQjz4jafMwnOnD1g9QMe8hlykSNUlCXNCbcA9DZG7COjPZHCY3Icv1+0Dp44bBTDQldR8DPACPMU5Qe+pWoTZViPgJQ7tq2V9TXrGdOnOtkPTwZ9KtZrmNhxW7tM=
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Middleman::Robots
|
2
2
|
|
3
3
|
[](http://badge.fury.io/rb/middleman-robots)
|
4
|
-
[](https://github.com/yterajima/middleman-robots/actions/workflows/ci.yml)
|
5
5
|
|
6
|
-
`middleman-robots` is an extension of [Middleman](
|
6
|
+
`middleman-robots` is an extension of [Middleman](https://middlemanapp.com/). This can create `robots.txt`.
|
7
7
|
|
8
8
|
This plugin support Middleman v3-stable and v4.
|
9
9
|
|
@@ -26,10 +26,10 @@ Or install it yourself as:
|
|
26
26
|
```ruby
|
27
27
|
# config.rb
|
28
28
|
activate :robots,
|
29
|
-
:
|
30
|
-
{:
|
29
|
+
rules: [
|
30
|
+
{ user_agent: '*', allow: %w[/] }
|
31
31
|
],
|
32
|
-
:
|
32
|
+
sitemap: 'https://example.com/sitemap.xml'
|
33
33
|
```
|
34
34
|
|
35
35
|
Created `robots.txt`:
|
@@ -38,28 +38,28 @@ Created `robots.txt`:
|
|
38
38
|
User-Agent: *
|
39
39
|
Allow: /
|
40
40
|
|
41
|
-
Sitemap:
|
41
|
+
Sitemap: https://example.com/sitemap.xml
|
42
42
|
|
43
43
|
```
|
44
44
|
|
45
|
-
You can use options,
|
45
|
+
You can use options, `rules` [ {`user_agent`(string), `allow`(array), `disallow`(array)}] and `sitemap`. Like this:
|
46
46
|
|
47
47
|
```ruby
|
48
48
|
# config.rb
|
49
49
|
activate :robots,
|
50
|
-
:
|
50
|
+
rules: [
|
51
51
|
{
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
52
|
+
user_agent: 'Googlebot',
|
53
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
54
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
55
55
|
},
|
56
56
|
{
|
57
|
-
:
|
58
|
-
:
|
59
|
-
:
|
57
|
+
user_agent: 'Googlebot-Image',
|
58
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
59
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
60
60
|
}
|
61
61
|
],
|
62
|
-
:
|
62
|
+
sitemap: 'https://example.com/sitemap.xml'
|
63
63
|
```
|
64
64
|
|
65
65
|
Created `robots.txt`:
|
@@ -77,7 +77,7 @@ Disallow: /something/dir/file_disallow.html
|
|
77
77
|
Allow: /allow/
|
78
78
|
Allow: /something/dir/file_allow.html
|
79
79
|
|
80
|
-
Sitemap:
|
80
|
+
Sitemap: https://example.com/sitemap.xml
|
81
81
|
|
82
82
|
```
|
83
83
|
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'bundler/gem_tasks'
|
2
4
|
|
3
5
|
task default: :test
|
4
6
|
|
5
7
|
desc 'test command'
|
6
8
|
task :test do
|
7
|
-
|
8
|
-
sh "bundle exec ruby #{f}"
|
9
|
-
end
|
9
|
+
sh 'bundle exec rspec spec/'
|
10
10
|
sh 'bundle exec cucumber features/'
|
11
11
|
end
|
data/cucumber.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
default: --publish-quiet
|
data/features/build.feature
CHANGED
@@ -9,7 +9,6 @@ Feature: Middleman-Robots on build
|
|
9
9
|
And a successfully built app at "basic-app"
|
10
10
|
When I cd to "build"
|
11
11
|
Then a file named "robots.txt" should exist
|
12
|
-
And the output should contain "== middleman-robots: robots.txt added to resources =="
|
13
12
|
And the file "robots.txt" should contain exactly:
|
14
13
|
"""
|
15
14
|
"""
|
@@ -18,9 +17,7 @@ Feature: Middleman-Robots on build
|
|
18
17
|
Given a fixture app "basic-app"
|
19
18
|
And a file named "config.rb" with:
|
20
19
|
"""
|
21
|
-
activate :robots, :
|
22
|
-
{:user_agent => '*'}
|
23
|
-
]
|
20
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
24
21
|
"""
|
25
22
|
And a successfully built app at "basic-app"
|
26
23
|
When I cd to "build"
|
@@ -34,9 +31,7 @@ Feature: Middleman-Robots on build
|
|
34
31
|
And a file named "config.rb" with:
|
35
32
|
"""
|
36
33
|
activate :robots do |r|
|
37
|
-
r.rules = [
|
38
|
-
{:user_agent => '*'}
|
39
|
-
]
|
34
|
+
r.rules = [ { user_agent: '*'} ]
|
40
35
|
end
|
41
36
|
"""
|
42
37
|
And a successfully built app at "basic-app"
|
@@ -50,9 +45,7 @@ Feature: Middleman-Robots on build
|
|
50
45
|
Given a fixture app "basic-app"
|
51
46
|
And a file named "config.rb" with:
|
52
47
|
"""
|
53
|
-
activate :robots, :
|
54
|
-
{:user_agent => '*'}
|
55
|
-
]
|
48
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
56
49
|
"""
|
57
50
|
And a successfully built app at "basic-app"
|
58
51
|
When I cd to "build"
|
@@ -65,9 +58,7 @@ Feature: Middleman-Robots on build
|
|
65
58
|
Given a fixture app "basic-app"
|
66
59
|
And a file named "config.rb" with:
|
67
60
|
"""
|
68
|
-
activate :robots, :
|
69
|
-
{'user-agent' => '*'}
|
70
|
-
]
|
61
|
+
activate :robots, rules: [ { 'user-agent': '*' } ]
|
71
62
|
"""
|
72
63
|
And a successfully built app at "basic-app"
|
73
64
|
When I cd to "build"
|
@@ -80,10 +71,10 @@ Feature: Middleman-Robots on build
|
|
80
71
|
Given a fixture app "basic-app"
|
81
72
|
And a file named "config.rb" with:
|
82
73
|
"""
|
83
|
-
activate :robots, :
|
74
|
+
activate :robots, rules: [
|
84
75
|
{
|
85
|
-
:
|
86
|
-
:
|
76
|
+
user_agent: '*',
|
77
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html]
|
87
78
|
}
|
88
79
|
]
|
89
80
|
"""
|
@@ -100,10 +91,10 @@ Feature: Middleman-Robots on build
|
|
100
91
|
Given a fixture app "basic-app"
|
101
92
|
And a file named "config.rb" with:
|
102
93
|
"""
|
103
|
-
activate :robots, :
|
94
|
+
activate :robots, rules: [
|
104
95
|
{
|
105
|
-
:
|
106
|
-
:
|
96
|
+
user_agent: '*',
|
97
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
107
98
|
}
|
108
99
|
]
|
109
100
|
"""
|
@@ -120,11 +111,11 @@ Feature: Middleman-Robots on build
|
|
120
111
|
Given a fixture app "basic-app"
|
121
112
|
And a file named "config.rb" with:
|
122
113
|
"""
|
123
|
-
activate :robots, :
|
114
|
+
activate :robots, rules: [
|
124
115
|
{
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
116
|
+
user_agent: '*',
|
117
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
118
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
128
119
|
}
|
129
120
|
]
|
130
121
|
"""
|
@@ -143,16 +134,16 @@ Feature: Middleman-Robots on build
|
|
143
134
|
Given a fixture app "basic-app"
|
144
135
|
And a file named "config.rb" with:
|
145
136
|
"""
|
146
|
-
activate :robots, :
|
137
|
+
activate :robots, rules: [
|
147
138
|
{
|
148
|
-
:
|
149
|
-
:
|
150
|
-
:
|
139
|
+
user_agent: 'Googlebot',
|
140
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
141
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
151
142
|
},
|
152
143
|
{
|
153
|
-
:
|
154
|
-
:
|
155
|
-
:
|
144
|
+
user_agent: 'Googlebot-Image',
|
145
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
146
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
156
147
|
}
|
157
148
|
]
|
158
149
|
"""
|
@@ -177,7 +168,7 @@ Feature: Middleman-Robots on build
|
|
177
168
|
Given a fixture app "basic-app"
|
178
169
|
And a file named "config.rb" with:
|
179
170
|
"""
|
180
|
-
activate :robots, :
|
171
|
+
activate :robots, sitemap: 'http://example.com/sitemap.xml'
|
181
172
|
"""
|
182
173
|
And a successfully built app at "basic-app"
|
183
174
|
When I cd to "build"
|
@@ -191,19 +182,19 @@ Feature: Middleman-Robots on build
|
|
191
182
|
And a file named "config.rb" with:
|
192
183
|
"""
|
193
184
|
activate :robots,
|
194
|
-
:
|
185
|
+
rules: [
|
195
186
|
{
|
196
|
-
:
|
197
|
-
:
|
198
|
-
:
|
187
|
+
user_agent:'Googlebot',
|
188
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
189
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
199
190
|
},
|
200
191
|
{
|
201
|
-
:
|
202
|
-
:
|
203
|
-
:
|
192
|
+
user_agent:'Googlebot-Image',
|
193
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
194
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
204
195
|
}
|
205
196
|
],
|
206
|
-
:
|
197
|
+
sitemap: 'http://example.com/sitemap.xml'
|
207
198
|
"""
|
208
199
|
And a successfully built app at "basic-app"
|
209
200
|
When I cd to "build"
|
data/features/support/env.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
PROJECT_ROOT_PATH = File.expand_path('../../', __dir__)
|
2
4
|
require 'middleman-core'
|
3
5
|
require 'middleman-core/step_definitions'
|
4
6
|
require File.join(PROJECT_ROOT_PATH, 'lib', 'middleman-robots')
|
5
|
-
|
@@ -1,14 +1,28 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
rules = [
|
4
|
+
{
|
5
|
+
user_agent: 'Googlebot',
|
6
|
+
disallow: %w[
|
7
|
+
tmp/*
|
8
|
+
/something/dir/file_disallow.html
|
9
|
+
],
|
10
|
+
allow: %w[
|
11
|
+
allow/*
|
12
|
+
/something/dir/file_allow.html
|
13
|
+
]
|
14
|
+
},
|
15
|
+
{
|
16
|
+
user_agent: 'Googlebot-Image',
|
17
|
+
disallow: %w[
|
18
|
+
tmp/*
|
19
|
+
/something/dir/file_disallow.html
|
20
|
+
],
|
21
|
+
allow: %w[
|
22
|
+
allow/*
|
23
|
+
/something/dir/file_allow.html
|
24
|
+
]
|
25
|
+
}
|
26
|
+
]
|
27
|
+
|
28
|
+
activate :robots, rules: rules, sitemap: 'http://example.com/sitemap.xml'
|
data/lib/middleman-robots.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'middleman-robots/generator'
|
2
4
|
require 'tempfile'
|
3
5
|
|
@@ -9,19 +11,22 @@ module Middleman
|
|
9
11
|
option :sitemap, false, 'URI of sitemap.xml'
|
10
12
|
|
11
13
|
def manipulate_resource_list(resources)
|
12
|
-
|
13
|
-
|
14
|
-
f.puts(Generator.new(options.rules, options.sitemap).process)
|
15
|
-
end
|
16
|
-
|
17
|
-
robots = Middleman::Sitemap::Resource.new(
|
14
|
+
write_robots(options)
|
15
|
+
resources << Middleman::Sitemap::Resource.new(
|
18
16
|
app.sitemap,
|
19
17
|
'robots.txt',
|
20
18
|
tmp_path
|
21
19
|
)
|
20
|
+
end
|
21
|
+
|
22
|
+
def write_robots(options)
|
23
|
+
File.open(tmp_path, 'w+') do |f|
|
24
|
+
f.puts(Generator.new(options.rules, options.sitemap).process)
|
25
|
+
end
|
26
|
+
end
|
22
27
|
|
23
|
-
|
24
|
-
|
28
|
+
def tmp_path
|
29
|
+
File.expand_path('../../tmp/robots/robots.txt', __dir__)
|
25
30
|
end
|
26
31
|
end
|
27
32
|
end
|