middleman-robots 1.3.2 → 1.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +25 -0
- data/.rspec +2 -0
- data/.rubocop.yml +28 -0
- data/.travis.yml +8 -9
- data/Gemfile +2 -0
- data/README.md +16 -16
- data/Rakefile +3 -3
- data/cucumber.yml +1 -0
- data/features/build.feature +30 -39
- data/features/support/env.rb +3 -2
- data/fixtures/server-app/config.rb +28 -14
- data/lib/middleman-robots.rb +4 -2
- data/lib/middleman-robots/extension.rb +13 -8
- data/lib/middleman-robots/generator.rb +12 -26
- data/lib/middleman-robots/generators/block.rb +55 -0
- data/lib/middleman-robots/generators/blocks.rb +26 -0
- data/lib/middleman-robots/generators/sitemap_uri.rb +24 -0
- data/lib/middleman-robots/version.rb +3 -1
- data/lib/middleman_extension.rb +2 -1
- data/middleman-robots.gemspec +21 -18
- data/spec/lib/middleman-robots/generator_spec.rb +65 -0
- data/spec/lib/middleman-robots/generators/block_spec.rb +205 -0
- data/spec/lib/middleman-robots/generators/blocks_spec.rb +50 -0
- data/spec/lib/middleman-robots/generators/sitemap_uri_spec.rb +51 -0
- data/spec/spec_helper.rb +13 -0
- metadata +97 -29
- data/lib/middleman-robots/group.rb +0 -50
- data/tests/test_generator.rb +0 -37
- data/tests/test_group.rb +0 -121
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 525009b32d148d32e07a4373e7532a46126e2b271ffef6221ad4d750d5a9b4a9
|
4
|
+
data.tar.gz: 45366b77e1e3ca26d889d4cead667c3826387063c48a5ae97dd12cde7f470d79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a9826c1a46025521f1fb884a8c17581e9d3aa9b3226fe2e37487ba1b33fd70c978c52a9b0d05164ee288e4d3d2c30597671c4c0a6809be94c72cea5f9821c31b
|
7
|
+
data.tar.gz: f152418cb899e8874ef4fbe52a79b785bb3cb9c3b7d08adb37c444f035f2e8c87319a977aa0e095b0b4683d909da6d9031fc1a46835c4c9601f49930634089d3
|
@@ -0,0 +1,25 @@
|
|
1
|
+
on: [push]
|
2
|
+
name: CI
|
3
|
+
jobs:
|
4
|
+
test:
|
5
|
+
strategy:
|
6
|
+
matrix:
|
7
|
+
ruby-version: [2.6, 2.7, 3.0]
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
steps:
|
10
|
+
- uses: actions/checkout@v2
|
11
|
+
- uses: ruby/setup-ruby@v1
|
12
|
+
with:
|
13
|
+
ruby-version: ${{ matrix.ruby-version }}
|
14
|
+
bundler-cache: true
|
15
|
+
- run: bundle exec rake test
|
16
|
+
|
17
|
+
rubocop:
|
18
|
+
runs-on: ubuntu-latest
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
- uses: ruby/setup-ruby@v1
|
22
|
+
with:
|
23
|
+
ruby-version: 3.0
|
24
|
+
bundler-cache: true
|
25
|
+
- run: bundle exec rubocop
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
AllCops:
|
2
|
+
NewCops: enable
|
3
|
+
Exclude:
|
4
|
+
- 'pkg/*'
|
5
|
+
- 'tmp/**/*'
|
6
|
+
- 'vendor/**/*'
|
7
|
+
TargetRubyVersion: 2.6
|
8
|
+
DisplayCopNames: true
|
9
|
+
|
10
|
+
Style/ClassAndModuleChildren:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Style/Documentation:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Metrics/BlockLength:
|
17
|
+
Exclude:
|
18
|
+
- 'spec/**/*_spec.rb'
|
19
|
+
|
20
|
+
Layout/LineLength:
|
21
|
+
Max: 100
|
22
|
+
Exclude:
|
23
|
+
- '*.gemspec'
|
24
|
+
|
25
|
+
Naming/FileName:
|
26
|
+
Enabled: true
|
27
|
+
Exclude:
|
28
|
+
- 'lib/middleman-robots.rb'
|
data/.travis.yml
CHANGED
@@ -5,18 +5,17 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.4
|
9
|
-
- 2.
|
10
|
-
- 2.
|
8
|
+
- 2.6.4
|
9
|
+
- 2.5.6
|
10
|
+
- 2.4.7
|
11
11
|
os:
|
12
|
-
- linux
|
12
|
+
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
|
-
- rvm:
|
17
|
-
script:
|
16
|
+
- rvm: ruby-head
|
17
|
+
script:
|
18
|
+
- bundle exec rubocop
|
19
|
+
- bundle exec rake test
|
18
20
|
cache: bundler
|
19
21
|
env: TEST=true
|
20
|
-
notifications:
|
21
|
-
slack:
|
22
|
-
secure: C8xb94J4fstzAI7t9GcvKQieKc1k+ECA96ydcxDwBI1fAbupQjz4jafMwnOnD1g9QMe8hlykSNUlCXNCbcA9DZG7COjPZHCY3Icv1+0Dp44bBTDQldR8DPACPMU5Qe+pWoTZViPgJQ7tq2V9TXrGdOnOtkPTwZ9KtZrmNhxW7tM=
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# Middleman::Robots
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/middleman-robots.svg)](http://badge.fury.io/rb/middleman-robots)
|
4
|
-
[![
|
4
|
+
[![GitHub Actions CI](https://github.com/yterajima/middleman-robots/actions/workflows/ci.yml/badge.svg)](https://github.com/yterajima/middleman-robots/actions/workflows/ci.yml)
|
5
5
|
|
6
|
-
`middleman-robots` is an extension of [Middleman](
|
6
|
+
`middleman-robots` is an extension of [Middleman](https://middlemanapp.com/). This can create `robots.txt`.
|
7
7
|
|
8
8
|
This plugin support Middleman v3-stable and v4.
|
9
9
|
|
@@ -26,10 +26,10 @@ Or install it yourself as:
|
|
26
26
|
```ruby
|
27
27
|
# config.rb
|
28
28
|
activate :robots,
|
29
|
-
:
|
30
|
-
{:
|
29
|
+
rules: [
|
30
|
+
{ user_agent: '*', allow: %w[/] }
|
31
31
|
],
|
32
|
-
:
|
32
|
+
sitemap: 'https://example.com/sitemap.xml'
|
33
33
|
```
|
34
34
|
|
35
35
|
Created `robots.txt`:
|
@@ -38,28 +38,28 @@ Created `robots.txt`:
|
|
38
38
|
User-Agent: *
|
39
39
|
Allow: /
|
40
40
|
|
41
|
-
Sitemap:
|
41
|
+
Sitemap: https://example.com/sitemap.xml
|
42
42
|
|
43
43
|
```
|
44
44
|
|
45
|
-
You can use options,
|
45
|
+
You can use options, `rules` [ {`user_agent`(string), `allow`(array), `disallow`(array)}] and `sitemap`. Like this:
|
46
46
|
|
47
47
|
```ruby
|
48
48
|
# config.rb
|
49
49
|
activate :robots,
|
50
|
-
:
|
50
|
+
rules: [
|
51
51
|
{
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
52
|
+
user_agent: 'Googlebot',
|
53
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
54
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
55
55
|
},
|
56
56
|
{
|
57
|
-
:
|
58
|
-
:
|
59
|
-
:
|
57
|
+
user_agent: 'Googlebot-Image',
|
58
|
+
disallow: %w[tmp/ /something/dir/file_disallow.html],
|
59
|
+
allow: %w[allow/ /something/dir/file_allow.html]
|
60
60
|
}
|
61
61
|
],
|
62
|
-
:
|
62
|
+
sitemap: 'https://example.com/sitemap.xml'
|
63
63
|
```
|
64
64
|
|
65
65
|
Created `robots.txt`:
|
@@ -77,7 +77,7 @@ Disallow: /something/dir/file_disallow.html
|
|
77
77
|
Allow: /allow/
|
78
78
|
Allow: /something/dir/file_allow.html
|
79
79
|
|
80
|
-
Sitemap:
|
80
|
+
Sitemap: https://example.com/sitemap.xml
|
81
81
|
|
82
82
|
```
|
83
83
|
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'bundler/gem_tasks'
|
2
4
|
|
3
5
|
task default: :test
|
4
6
|
|
5
7
|
desc 'test command'
|
6
8
|
task :test do
|
7
|
-
|
8
|
-
sh "bundle exec ruby #{f}"
|
9
|
-
end
|
9
|
+
sh 'bundle exec rspec spec/'
|
10
10
|
sh 'bundle exec cucumber features/'
|
11
11
|
end
|
data/cucumber.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
default: --publish-quiet
|
data/features/build.feature
CHANGED
@@ -9,7 +9,6 @@ Feature: Middleman-Robots on build
|
|
9
9
|
And a successfully built app at "basic-app"
|
10
10
|
When I cd to "build"
|
11
11
|
Then a file named "robots.txt" should exist
|
12
|
-
And the output should contain "== middleman-robots: robots.txt added to resources =="
|
13
12
|
And the file "robots.txt" should contain exactly:
|
14
13
|
"""
|
15
14
|
"""
|
@@ -18,9 +17,7 @@ Feature: Middleman-Robots on build
|
|
18
17
|
Given a fixture app "basic-app"
|
19
18
|
And a file named "config.rb" with:
|
20
19
|
"""
|
21
|
-
activate :robots, :
|
22
|
-
{:user_agent => '*'}
|
23
|
-
]
|
20
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
24
21
|
"""
|
25
22
|
And a successfully built app at "basic-app"
|
26
23
|
When I cd to "build"
|
@@ -34,9 +31,7 @@ Feature: Middleman-Robots on build
|
|
34
31
|
And a file named "config.rb" with:
|
35
32
|
"""
|
36
33
|
activate :robots do |r|
|
37
|
-
r.rules = [
|
38
|
-
{:user_agent => '*'}
|
39
|
-
]
|
34
|
+
r.rules = [ { user_agent: '*'} ]
|
40
35
|
end
|
41
36
|
"""
|
42
37
|
And a successfully built app at "basic-app"
|
@@ -50,9 +45,7 @@ Feature: Middleman-Robots on build
|
|
50
45
|
Given a fixture app "basic-app"
|
51
46
|
And a file named "config.rb" with:
|
52
47
|
"""
|
53
|
-
activate :robots, :
|
54
|
-
{:user_agent => '*'}
|
55
|
-
]
|
48
|
+
activate :robots, rules: [ { user_agent: '*'} ]
|
56
49
|
"""
|
57
50
|
And a successfully built app at "basic-app"
|
58
51
|
When I cd to "build"
|
@@ -65,9 +58,7 @@ Feature: Middleman-Robots on build
|
|
65
58
|
Given a fixture app "basic-app"
|
66
59
|
And a file named "config.rb" with:
|
67
60
|
"""
|
68
|
-
activate :robots, :
|
69
|
-
{'user-agent' => '*'}
|
70
|
-
]
|
61
|
+
activate :robots, rules: [ { 'user-agent': '*' } ]
|
71
62
|
"""
|
72
63
|
And a successfully built app at "basic-app"
|
73
64
|
When I cd to "build"
|
@@ -80,10 +71,10 @@ Feature: Middleman-Robots on build
|
|
80
71
|
Given a fixture app "basic-app"
|
81
72
|
And a file named "config.rb" with:
|
82
73
|
"""
|
83
|
-
activate :robots, :
|
74
|
+
activate :robots, rules: [
|
84
75
|
{
|
85
|
-
:
|
86
|
-
:
|
76
|
+
user_agent: '*',
|
77
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html]
|
87
78
|
}
|
88
79
|
]
|
89
80
|
"""
|
@@ -100,10 +91,10 @@ Feature: Middleman-Robots on build
|
|
100
91
|
Given a fixture app "basic-app"
|
101
92
|
And a file named "config.rb" with:
|
102
93
|
"""
|
103
|
-
activate :robots, :
|
94
|
+
activate :robots, rules: [
|
104
95
|
{
|
105
|
-
:
|
106
|
-
:
|
96
|
+
user_agent: '*',
|
97
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
107
98
|
}
|
108
99
|
]
|
109
100
|
"""
|
@@ -120,11 +111,11 @@ Feature: Middleman-Robots on build
|
|
120
111
|
Given a fixture app "basic-app"
|
121
112
|
And a file named "config.rb" with:
|
122
113
|
"""
|
123
|
-
activate :robots, :
|
114
|
+
activate :robots, rules: [
|
124
115
|
{
|
125
|
-
:
|
126
|
-
:
|
127
|
-
:
|
116
|
+
user_agent: '*',
|
117
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
118
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
128
119
|
}
|
129
120
|
]
|
130
121
|
"""
|
@@ -143,16 +134,16 @@ Feature: Middleman-Robots on build
|
|
143
134
|
Given a fixture app "basic-app"
|
144
135
|
And a file named "config.rb" with:
|
145
136
|
"""
|
146
|
-
activate :robots, :
|
137
|
+
activate :robots, rules: [
|
147
138
|
{
|
148
|
-
:
|
149
|
-
:
|
150
|
-
:
|
139
|
+
user_agent: 'Googlebot',
|
140
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
141
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
151
142
|
},
|
152
143
|
{
|
153
|
-
:
|
154
|
-
:
|
155
|
-
:
|
144
|
+
user_agent: 'Googlebot-Image',
|
145
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
146
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
156
147
|
}
|
157
148
|
]
|
158
149
|
"""
|
@@ -177,7 +168,7 @@ Feature: Middleman-Robots on build
|
|
177
168
|
Given a fixture app "basic-app"
|
178
169
|
And a file named "config.rb" with:
|
179
170
|
"""
|
180
|
-
activate :robots, :
|
171
|
+
activate :robots, sitemap: 'http://example.com/sitemap.xml'
|
181
172
|
"""
|
182
173
|
And a successfully built app at "basic-app"
|
183
174
|
When I cd to "build"
|
@@ -191,19 +182,19 @@ Feature: Middleman-Robots on build
|
|
191
182
|
And a file named "config.rb" with:
|
192
183
|
"""
|
193
184
|
activate :robots,
|
194
|
-
:
|
185
|
+
rules: [
|
195
186
|
{
|
196
|
-
:
|
197
|
-
:
|
198
|
-
:
|
187
|
+
user_agent:'Googlebot',
|
188
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
189
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
199
190
|
},
|
200
191
|
{
|
201
|
-
:
|
202
|
-
:
|
203
|
-
:
|
192
|
+
user_agent:'Googlebot-Image',
|
193
|
+
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
194
|
+
allow: %w[allow/* /something/dir/file_allow.html]
|
204
195
|
}
|
205
196
|
],
|
206
|
-
:
|
197
|
+
sitemap: 'http://example.com/sitemap.xml'
|
207
198
|
"""
|
208
199
|
And a successfully built app at "basic-app"
|
209
200
|
When I cd to "build"
|
data/features/support/env.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
PROJECT_ROOT_PATH = File.expand_path('../../', __dir__)
|
2
4
|
require 'middleman-core'
|
3
5
|
require 'middleman-core/step_definitions'
|
4
6
|
require File.join(PROJECT_ROOT_PATH, 'lib', 'middleman-robots')
|
5
|
-
|
@@ -1,14 +1,28 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
rules = [
|
4
|
+
{
|
5
|
+
user_agent: 'Googlebot',
|
6
|
+
disallow: %w[
|
7
|
+
tmp/*
|
8
|
+
/something/dir/file_disallow.html
|
9
|
+
],
|
10
|
+
allow: %w[
|
11
|
+
allow/*
|
12
|
+
/something/dir/file_allow.html
|
13
|
+
]
|
14
|
+
},
|
15
|
+
{
|
16
|
+
user_agent: 'Googlebot-Image',
|
17
|
+
disallow: %w[
|
18
|
+
tmp/*
|
19
|
+
/something/dir/file_disallow.html
|
20
|
+
],
|
21
|
+
allow: %w[
|
22
|
+
allow/*
|
23
|
+
/something/dir/file_allow.html
|
24
|
+
]
|
25
|
+
}
|
26
|
+
]
|
27
|
+
|
28
|
+
activate :robots, rules: rules, sitemap: 'http://example.com/sitemap.xml'
|
data/lib/middleman-robots.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'middleman-robots/generator'
|
2
4
|
require 'tempfile'
|
3
5
|
|
@@ -9,19 +11,22 @@ module Middleman
|
|
9
11
|
option :sitemap, false, 'URI of sitemap.xml'
|
10
12
|
|
11
13
|
def manipulate_resource_list(resources)
|
12
|
-
|
13
|
-
|
14
|
-
f.puts(Generator.new(options.rules, options.sitemap).process)
|
15
|
-
end
|
16
|
-
|
17
|
-
robots = Middleman::Sitemap::Resource.new(
|
14
|
+
write_robots(options)
|
15
|
+
resources << Middleman::Sitemap::Resource.new(
|
18
16
|
app.sitemap,
|
19
17
|
'robots.txt',
|
20
18
|
tmp_path
|
21
19
|
)
|
20
|
+
end
|
21
|
+
|
22
|
+
def write_robots(options)
|
23
|
+
File.open(tmp_path, 'w+') do |f|
|
24
|
+
f.puts(Generator.new(options.rules, options.sitemap).process)
|
25
|
+
end
|
26
|
+
end
|
22
27
|
|
23
|
-
|
24
|
-
|
28
|
+
def tmp_path
|
29
|
+
File.expand_path('../../tmp/robots/robots.txt', __dir__)
|
25
30
|
end
|
26
31
|
end
|
27
32
|
end
|