middleman-robots 1.3.3 → 1.3.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +25 -0
- data/.rspec +2 -0
- data/.rubocop.yml +7 -17
- data/.travis.yml +5 -8
- data/Gemfile +2 -0
- data/README.md +1 -1
- data/Rakefile +3 -3
- data/cucumber.yml +1 -0
- data/features/build.feature +0 -1
- data/features/support/env.rb +3 -1
- data/fixtures/server-app/config.rb +18 -4
- data/lib/middleman-robots/extension.rb +4 -5
- data/lib/middleman-robots/generator.rb +12 -27
- data/lib/middleman-robots/generators/block.rb +55 -0
- data/lib/middleman-robots/generators/blocks.rb +26 -0
- data/lib/middleman-robots/generators/sitemap_uri.rb +24 -0
- data/lib/middleman-robots/version.rb +3 -1
- data/lib/middleman-robots.rb +2 -0
- data/lib/middleman_extension.rb +2 -0
- data/middleman-robots.gemspec +9 -3
- data/spec/lib/middleman-robots/generator_spec.rb +65 -0
- data/spec/lib/middleman-robots/generators/block_spec.rb +205 -0
- data/spec/lib/middleman-robots/generators/blocks_spec.rb +50 -0
- data/spec/lib/middleman-robots/generators/sitemap_uri_spec.rb +51 -0
- data/spec/spec_helper.rb +13 -0
- metadata +56 -15
- data/lib/middleman-robots/group.rb +0 -51
- data/tests/test_generator.rb +0 -43
- data/tests/test_group.rb +0 -120
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16f45acdedbbcb08a7dce95ad4051ea45b976a9a01e692210270dd656c583ef9
|
4
|
+
data.tar.gz: '079e7a6d4e9eb82f66d6d39f4003d4df5a92965e49d04341d0f83c16b4ec2cd6'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 78e9df4262088e82f9c71bc7d0d3de288674c638630d9e7a551a3f86addc7da30c6360a8bc2764159194afc1b3b9516b39e73a24b1af9fd0ca89c2b547b9cdfb
|
7
|
+
data.tar.gz: e338f1d12c7706eaddf471162fc60107a726eaff3fd7aa827deea65669661194c13445d51fc7f3fab1d89669f9669c13a7214706fef41c0ad2e89f371fc5e04c
|
@@ -0,0 +1,25 @@
|
|
1
|
+
on: [push]
|
2
|
+
name: CI
|
3
|
+
jobs:
|
4
|
+
test:
|
5
|
+
strategy:
|
6
|
+
matrix:
|
7
|
+
ruby-version: [2.6, 2.7, 3.0]
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
steps:
|
10
|
+
- uses: actions/checkout@v2
|
11
|
+
- uses: ruby/setup-ruby@v1
|
12
|
+
with:
|
13
|
+
ruby-version: ${{ matrix.ruby-version }}
|
14
|
+
bundler-cache: true
|
15
|
+
- run: bundle exec rake test
|
16
|
+
|
17
|
+
rubocop:
|
18
|
+
runs-on: ubuntu-latest
|
19
|
+
steps:
|
20
|
+
- uses: actions/checkout@v2
|
21
|
+
- uses: ruby/setup-ruby@v1
|
22
|
+
with:
|
23
|
+
ruby-version: 3.0
|
24
|
+
bundler-cache: true
|
25
|
+
- run: bundle exec rubocop
|
data/.rspec
ADDED
data/.rubocop.yml
CHANGED
@@ -1,37 +1,27 @@
|
|
1
1
|
AllCops:
|
2
|
+
NewCops: enable
|
2
3
|
Exclude:
|
3
4
|
- 'pkg/*'
|
4
5
|
- 'tmp/**/*'
|
5
6
|
- 'vendor/**/*'
|
6
|
-
TargetRubyVersion: 2.
|
7
|
+
TargetRubyVersion: 2.6
|
7
8
|
DisplayCopNames: true
|
8
9
|
|
9
|
-
Rails:
|
10
|
-
Enabled: false
|
11
|
-
|
12
10
|
Style/ClassAndModuleChildren:
|
13
11
|
Enabled: false
|
14
12
|
|
15
13
|
Style/Documentation:
|
16
14
|
Enabled: false
|
17
15
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
Style/RedundantSelf:
|
22
|
-
Enabled: false
|
23
|
-
|
24
|
-
Gemspec/RequiredRubyVersion:
|
25
|
-
Enabled: false
|
16
|
+
Metrics/BlockLength:
|
17
|
+
Exclude:
|
18
|
+
- 'spec/**/*_spec.rb'
|
26
19
|
|
27
|
-
|
28
|
-
Max:
|
20
|
+
Layout/LineLength:
|
21
|
+
Max: 100
|
29
22
|
Exclude:
|
30
23
|
- '*.gemspec'
|
31
24
|
|
32
|
-
Metrics/MethodLength:
|
33
|
-
Max: 20
|
34
|
-
|
35
25
|
Naming/FileName:
|
36
26
|
Enabled: true
|
37
27
|
Exclude:
|
data/.travis.yml
CHANGED
@@ -5,20 +5,17 @@ before_script:
|
|
5
5
|
- bundle update
|
6
6
|
rvm:
|
7
7
|
- ruby-head
|
8
|
-
- 2.
|
9
|
-
- 2.
|
10
|
-
- 2.
|
8
|
+
- 2.6.4
|
9
|
+
- 2.5.6
|
10
|
+
- 2.4.7
|
11
11
|
os:
|
12
|
-
- linux
|
12
|
+
- linux
|
13
13
|
matrix:
|
14
14
|
fast_finish: true
|
15
15
|
allow_failures:
|
16
16
|
- rvm: ruby-head
|
17
17
|
script:
|
18
|
-
- bundle exec rake test
|
19
18
|
- bundle exec rubocop
|
19
|
+
- bundle exec rake test
|
20
20
|
cache: bundler
|
21
21
|
env: TEST=true
|
22
|
-
notifications:
|
23
|
-
slack:
|
24
|
-
secure: C8xb94J4fstzAI7t9GcvKQieKc1k+ECA96ydcxDwBI1fAbupQjz4jafMwnOnD1g9QMe8hlykSNUlCXNCbcA9DZG7COjPZHCY3Icv1+0Dp44bBTDQldR8DPACPMU5Qe+pWoTZViPgJQ7tq2V9TXrGdOnOtkPTwZ9KtZrmNhxW7tM=
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Middleman::Robots
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/middleman-robots.svg)](http://badge.fury.io/rb/middleman-robots)
|
4
|
-
[![
|
4
|
+
[![GitHub Actions CI](https://github.com/yterajima/middleman-robots/actions/workflows/ci.yml/badge.svg)](https://github.com/yterajima/middleman-robots/actions/workflows/ci.yml)
|
5
5
|
|
6
6
|
`middleman-robots` is an extension of [Middleman](https://middlemanapp.com/). This can create `robots.txt`.
|
7
7
|
|
data/Rakefile
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'bundler/gem_tasks'
|
2
4
|
|
3
5
|
task default: :test
|
4
6
|
|
5
7
|
desc 'test command'
|
6
8
|
task :test do
|
7
|
-
|
8
|
-
sh "bundle exec ruby #{f}"
|
9
|
-
end
|
9
|
+
sh 'bundle exec rspec spec/'
|
10
10
|
sh 'bundle exec cucumber features/'
|
11
11
|
end
|
data/cucumber.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
default: --publish-quiet
|
data/features/build.feature
CHANGED
@@ -9,7 +9,6 @@ Feature: Middleman-Robots on build
|
|
9
9
|
And a successfully built app at "basic-app"
|
10
10
|
When I cd to "build"
|
11
11
|
Then a file named "robots.txt" should exist
|
12
|
-
And the output should contain "== middleman-robots: robots.txt added to resources =="
|
13
12
|
And the file "robots.txt" should contain exactly:
|
14
13
|
"""
|
15
14
|
"""
|
data/features/support/env.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
PROJECT_ROOT_PATH = File.expand_path('../../', __dir__)
|
2
4
|
require 'middleman-core'
|
3
5
|
require 'middleman-core/step_definitions'
|
4
6
|
require File.join(PROJECT_ROOT_PATH, 'lib', 'middleman-robots')
|
@@ -1,13 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
rules = [
|
2
4
|
{
|
3
5
|
user_agent: 'Googlebot',
|
4
|
-
disallow: %w[
|
5
|
-
|
6
|
+
disallow: %w[
|
7
|
+
tmp/*
|
8
|
+
/something/dir/file_disallow.html
|
9
|
+
],
|
10
|
+
allow: %w[
|
11
|
+
allow/*
|
12
|
+
/something/dir/file_allow.html
|
13
|
+
]
|
6
14
|
},
|
7
15
|
{
|
8
16
|
user_agent: 'Googlebot-Image',
|
9
|
-
disallow:
|
10
|
-
|
17
|
+
disallow: %w[
|
18
|
+
tmp/*
|
19
|
+
/something/dir/file_disallow.html
|
20
|
+
],
|
21
|
+
allow: %w[
|
22
|
+
allow/*
|
23
|
+
/something/dir/file_allow.html
|
24
|
+
]
|
11
25
|
}
|
12
26
|
]
|
13
27
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'middleman-robots/generator'
|
2
4
|
require 'tempfile'
|
3
5
|
|
@@ -10,14 +12,11 @@ module Middleman
|
|
10
12
|
|
11
13
|
def manipulate_resource_list(resources)
|
12
14
|
write_robots(options)
|
13
|
-
|
15
|
+
resources << Middleman::Sitemap::Resource.new(
|
14
16
|
app.sitemap,
|
15
17
|
'robots.txt',
|
16
18
|
tmp_path
|
17
19
|
)
|
18
|
-
|
19
|
-
logger.info '== middleman-robots: robots.txt added to resources =='
|
20
|
-
resources << robots
|
21
20
|
end
|
22
21
|
|
23
22
|
def write_robots(options)
|
@@ -27,7 +26,7 @@ module Middleman
|
|
27
26
|
end
|
28
27
|
|
29
28
|
def tmp_path
|
30
|
-
File.expand_path('
|
29
|
+
File.expand_path('../../tmp/robots/robots.txt', __dir__)
|
31
30
|
end
|
32
31
|
end
|
33
32
|
end
|
@@ -1,42 +1,27 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/blocks'
|
4
|
+
require 'middleman-robots/generators/sitemap_uri'
|
2
5
|
|
3
6
|
module Middleman
|
4
7
|
module Robots
|
5
8
|
# Robots Text Generator Class
|
6
9
|
class Generator
|
10
|
+
attr_accessor :rules, :sitemap_uri
|
11
|
+
|
7
12
|
def initialize(rules, sitemap_uri)
|
8
13
|
@rules = rules
|
9
14
|
@sitemap_uri = sitemap_uri
|
10
15
|
end
|
11
16
|
|
12
17
|
def process
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
blocks + "\n" + sitemap
|
18
|
-
elsif !blocks.empty?
|
19
|
-
blocks
|
20
|
-
elsif !sitemap.empty?
|
21
|
-
sitemap
|
22
|
-
else
|
23
|
-
''
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
def block_text
|
30
|
-
return '' if @rules.empty?
|
31
|
-
data = []
|
32
|
-
@rules.each do |rule|
|
33
|
-
data << Group.new(rule).text
|
34
|
-
end
|
35
|
-
data.join("\n")
|
36
|
-
end
|
18
|
+
text = [
|
19
|
+
Generators::Blocks.new(rules).text,
|
20
|
+
Generators::SitemapUri.new(sitemap_uri).text
|
21
|
+
].compact.join "\n\n"
|
37
22
|
|
38
|
-
|
39
|
-
|
23
|
+
text += "\n" if text.present?
|
24
|
+
text
|
40
25
|
end
|
41
26
|
end
|
42
27
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
# Block
|
9
|
+
#
|
10
|
+
# Generating Block in robots.txt
|
11
|
+
class Block
|
12
|
+
attr_accessor :rule
|
13
|
+
|
14
|
+
def initialize(rule)
|
15
|
+
@rule = rule
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
[
|
20
|
+
user_agent,
|
21
|
+
disallow,
|
22
|
+
allow
|
23
|
+
].compact.join("\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def user_agent
|
29
|
+
user_agent = rule[:user_agent].presence || rule['user-agent'].presence || '*'
|
30
|
+
if !user_agent.is_a?(String) && !user_agent.nil?
|
31
|
+
raise ArgumentError, '`user_agent` or `user-agent` option must be String or nil'
|
32
|
+
end
|
33
|
+
|
34
|
+
"User-Agent: #{user_agent}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def disallow
|
38
|
+
return nil if rule[:disallow].nil?
|
39
|
+
unless rule[:disallow].is_a? Array
|
40
|
+
raise ArgumentError, '`disallow` option must be Array or nil'
|
41
|
+
end
|
42
|
+
|
43
|
+
rule[:disallow].map { |path| "Disallow: #{File.join('/', path)}" }
|
44
|
+
end
|
45
|
+
|
46
|
+
def allow
|
47
|
+
return nil if rule[:allow].nil?
|
48
|
+
raise ArgumentError, '`allow` option must be Array or nil' unless rule[:allow].is_a? Array
|
49
|
+
|
50
|
+
rule[:allow].map { |path| "Allow: #{File.join('/', path)}" }
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
require 'middleman-robots/generators/block'
|
5
|
+
|
6
|
+
module Middleman
|
7
|
+
module Robots
|
8
|
+
module Generators
|
9
|
+
# Blocks
|
10
|
+
#
|
11
|
+
# Collection of ::Middleman::Robots::Generators::Block
|
12
|
+
class Blocks
|
13
|
+
def initialize(rules)
|
14
|
+
@rules = rules
|
15
|
+
@groups = @rules.map { |rule| Block.new(rule) } if @rules.present?
|
16
|
+
end
|
17
|
+
|
18
|
+
def text
|
19
|
+
return nil if @groups.nil?
|
20
|
+
|
21
|
+
@groups.map(&:text).join "\n\n"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
|
+
|
5
|
+
module Middleman
|
6
|
+
module Robots
|
7
|
+
module Generators
|
8
|
+
class SitemapUri
|
9
|
+
attr_accessor :uri
|
10
|
+
|
11
|
+
def initialize(uri)
|
12
|
+
@uri = uri
|
13
|
+
end
|
14
|
+
|
15
|
+
def text
|
16
|
+
return nil if uri.blank?
|
17
|
+
raise ArgumentError, 'sitemap_uri must be string or nil' unless uri.is_a? String
|
18
|
+
|
19
|
+
"Sitemap: #{uri}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/middleman-robots.rb
CHANGED
data/lib/middleman_extension.rb
CHANGED
data/middleman-robots.gemspec
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
2
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
5
|
require 'middleman-robots/version'
|
4
6
|
|
@@ -16,14 +18,18 @@ Gem::Specification.new do |spec|
|
|
16
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
20
|
spec.require_paths = ['lib']
|
19
|
-
spec.required_ruby_version = '>= 2.
|
21
|
+
spec.required_ruby_version = '>= 2.6.0'
|
20
22
|
|
21
|
-
spec.add_runtime_dependency 'middleman', '
|
23
|
+
spec.add_runtime_dependency 'middleman-cli', '~> 4.0'
|
24
|
+
spec.add_runtime_dependency 'middleman-core', '~> 4.0'
|
22
25
|
|
23
26
|
spec.add_development_dependency 'aruba', '>= 0.14.3'
|
24
27
|
spec.add_development_dependency 'bundler', '>= 1.16'
|
25
28
|
spec.add_development_dependency 'capybara', '>= 2.18.0'
|
26
29
|
spec.add_development_dependency 'cucumber', '>= 3.1.0'
|
27
30
|
spec.add_development_dependency 'rake', '>= 12.3'
|
31
|
+
spec.add_development_dependency 'rspec'
|
28
32
|
spec.add_development_dependency 'rubocop', '>= 0.52.1'
|
33
|
+
|
34
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
29
35
|
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generator'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generator do
|
6
|
+
describe '#process' do
|
7
|
+
subject { Middleman::Robots::Generator.new(rules, sitemap_uri).process }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rules) do
|
11
|
+
[
|
12
|
+
{
|
13
|
+
user_agent: 'Googlebot',
|
14
|
+
disallow: %w[
|
15
|
+
tmp/*
|
16
|
+
/something/dir/file_disallow.html
|
17
|
+
],
|
18
|
+
allow: %w[
|
19
|
+
allow/*
|
20
|
+
/something/dir/file_allow.html
|
21
|
+
]
|
22
|
+
},
|
23
|
+
{
|
24
|
+
user_agent: 'Googlebot-Image',
|
25
|
+
disallow: %w[
|
26
|
+
tmp/*
|
27
|
+
/something/dir/file_disallow.html
|
28
|
+
],
|
29
|
+
allow: %w[
|
30
|
+
allow/*
|
31
|
+
/something/dir/file_allow.html
|
32
|
+
]
|
33
|
+
}
|
34
|
+
]
|
35
|
+
end
|
36
|
+
let(:sitemap_uri) { 'http://example.com/sitemap.xml' }
|
37
|
+
let(:expected) do
|
38
|
+
<<~ROBOTS
|
39
|
+
User-Agent: Googlebot
|
40
|
+
Disallow: /tmp/*
|
41
|
+
Disallow: /something/dir/file_disallow.html
|
42
|
+
Allow: /allow/*
|
43
|
+
Allow: /something/dir/file_allow.html
|
44
|
+
|
45
|
+
User-Agent: Googlebot-Image
|
46
|
+
Disallow: /tmp/*
|
47
|
+
Disallow: /something/dir/file_disallow.html
|
48
|
+
Allow: /allow/*
|
49
|
+
Allow: /something/dir/file_allow.html
|
50
|
+
|
51
|
+
Sitemap: http://example.com/sitemap.xml
|
52
|
+
ROBOTS
|
53
|
+
end
|
54
|
+
|
55
|
+
it { is_expected.to eq expected }
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'without options' do
|
59
|
+
let(:rules) { nil }
|
60
|
+
let(:sitemap_uri) { nil }
|
61
|
+
|
62
|
+
it { is_expected.to be_empty }
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,205 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/block'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::Block do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(rule).text }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rule) do
|
11
|
+
{
|
12
|
+
user_agent: 'GoogleBot',
|
13
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
14
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
15
|
+
}
|
16
|
+
end
|
17
|
+
let(:expected) do
|
18
|
+
expected = <<~ROBOTS
|
19
|
+
User-Agent: GoogleBot
|
20
|
+
Disallow: /tmp/*
|
21
|
+
Disallow: /someting/dir/disallow.html
|
22
|
+
Allow: /allow/*
|
23
|
+
Allow: /someting/dir/allow.html
|
24
|
+
ROBOTS
|
25
|
+
expected.chomp
|
26
|
+
end
|
27
|
+
|
28
|
+
it { is_expected.to eq expected }
|
29
|
+
end
|
30
|
+
|
31
|
+
context 'without option' do
|
32
|
+
let(:rule) { {} }
|
33
|
+
|
34
|
+
it { is_expected.to eq 'User-Agent: *' }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with user_agent value is' do
|
38
|
+
context 'Middleman-Bot' do
|
39
|
+
let(:rule) { { user_agent: 'Middleman-Bot' } }
|
40
|
+
|
41
|
+
it { is_expected.to eq "User-Agent: #{rule[:user_agent]}" }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'nil' do
|
45
|
+
let(:rule) { { user_agent: nil } }
|
46
|
+
|
47
|
+
it { is_expected.to eq 'User-Agent: *' }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'empty' do
|
51
|
+
let(:rule) { { user_agent: '' } }
|
52
|
+
|
53
|
+
it { is_expected.to eq 'User-Agent: *' }
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'ERROR when' do
|
57
|
+
subject { -> { described_class.new(rule).text } }
|
58
|
+
|
59
|
+
context 'Array' do
|
60
|
+
let(:rule) { { user_agent: %w[a b] } }
|
61
|
+
|
62
|
+
it { is_expected.to raise_error ArgumentError }
|
63
|
+
end
|
64
|
+
|
65
|
+
context 'Numeric' do
|
66
|
+
let(:rule) { { user_agent: 1_000 } }
|
67
|
+
|
68
|
+
it { is_expected.to raise_error ArgumentError }
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'Symbol' do
|
72
|
+
let(:rule) { { user_agent: :user_agent } }
|
73
|
+
|
74
|
+
it { is_expected.to raise_error ArgumentError }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'with disallow value is' do
|
80
|
+
context 'single Array' do
|
81
|
+
let(:rule) { { disallow: %w[/tmp/*] } }
|
82
|
+
let(:expected) do
|
83
|
+
expected = <<~ROBOTS
|
84
|
+
User-Agent: *
|
85
|
+
Disallow: /tmp/*
|
86
|
+
ROBOTS
|
87
|
+
expected.chomp
|
88
|
+
end
|
89
|
+
|
90
|
+
it { is_expected.to eq expected }
|
91
|
+
end
|
92
|
+
|
93
|
+
context 'multiple Array' do
|
94
|
+
let(:rule) { { disallow: %w[/tmp/* /something/disallow.html] } }
|
95
|
+
let(:expected) do
|
96
|
+
expected = <<~ROBOTS
|
97
|
+
User-Agent: *
|
98
|
+
Disallow: /tmp/*
|
99
|
+
Disallow: /something/disallow.html
|
100
|
+
ROBOTS
|
101
|
+
expected.chomp
|
102
|
+
end
|
103
|
+
|
104
|
+
it { is_expected.to eq expected }
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'nil' do
|
108
|
+
let(:rule) { { disallow: nil } }
|
109
|
+
|
110
|
+
it { is_expected.to eq 'User-Agent: *' }
|
111
|
+
end
|
112
|
+
|
113
|
+
context 'ERROR when' do
|
114
|
+
subject { -> { described_class.new(rule).text } }
|
115
|
+
|
116
|
+
context 'empty' do
|
117
|
+
let(:rule) { { disallow: '' } }
|
118
|
+
|
119
|
+
it { is_expected.to raise_error ArgumentError }
|
120
|
+
end
|
121
|
+
|
122
|
+
context 'String' do
|
123
|
+
let(:rule) { { disallow: 'string' } }
|
124
|
+
|
125
|
+
it { is_expected.to raise_error ArgumentError }
|
126
|
+
end
|
127
|
+
|
128
|
+
context 'Numeric' do
|
129
|
+
let(:rule) { { disallow: 1_000 } }
|
130
|
+
|
131
|
+
it { is_expected.to raise_error ArgumentError }
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'Symbol' do
|
135
|
+
let(:rule) { { disallow: :disallow } }
|
136
|
+
|
137
|
+
it { is_expected.to raise_error ArgumentError }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context 'when allow value is' do
|
143
|
+
context 'single Array' do
|
144
|
+
let(:rule) { { allow: %w[/tmp/*] } }
|
145
|
+
let(:expected) do
|
146
|
+
expected = <<~ROBOTS
|
147
|
+
User-Agent: *
|
148
|
+
Allow: /tmp/*
|
149
|
+
ROBOTS
|
150
|
+
expected.chomp
|
151
|
+
end
|
152
|
+
|
153
|
+
it { is_expected.to eq expected }
|
154
|
+
end
|
155
|
+
|
156
|
+
context 'multiple Array' do
|
157
|
+
let(:rule) { { allow: %w[/tmp/* /something/allow.html] } }
|
158
|
+
let(:expected) do
|
159
|
+
expected = <<~ROBOTS
|
160
|
+
User-Agent: *
|
161
|
+
Allow: /tmp/*
|
162
|
+
Allow: /something/allow.html
|
163
|
+
ROBOTS
|
164
|
+
expected.chomp
|
165
|
+
end
|
166
|
+
|
167
|
+
it { is_expected.to eq expected }
|
168
|
+
end
|
169
|
+
|
170
|
+
context 'nil' do
|
171
|
+
let(:rule) { { allow: nil } }
|
172
|
+
|
173
|
+
it { is_expected.to eq 'User-Agent: *' }
|
174
|
+
end
|
175
|
+
|
176
|
+
context 'ERROR when' do
|
177
|
+
subject { -> { described_class.new(rule).text } }
|
178
|
+
|
179
|
+
context 'empty' do
|
180
|
+
let(:rule) { { allow: '' } }
|
181
|
+
|
182
|
+
it { is_expected.to raise_error ArgumentError }
|
183
|
+
end
|
184
|
+
|
185
|
+
context 'String' do
|
186
|
+
let(:rule) { { allow: 'string' } }
|
187
|
+
|
188
|
+
it { is_expected.to raise_error ArgumentError }
|
189
|
+
end
|
190
|
+
|
191
|
+
context 'Numeric' do
|
192
|
+
let(:rule) { { allow: 1_000 } }
|
193
|
+
|
194
|
+
it { is_expected.to raise_error ArgumentError }
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'Symbol' do
|
198
|
+
let(:rule) { { allow: :disallow } }
|
199
|
+
|
200
|
+
it { is_expected.to raise_error ArgumentError }
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/blocks'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::Blocks do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(rules).text }
|
8
|
+
|
9
|
+
context 'with all options' do
|
10
|
+
let(:rules) do
|
11
|
+
[
|
12
|
+
{
|
13
|
+
user_agent: 'GoogleBot',
|
14
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
15
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
16
|
+
},
|
17
|
+
{
|
18
|
+
user_agent: 'GoogleBot-Image',
|
19
|
+
disallow: %w[tmp/* /someting/dir/disallow.html],
|
20
|
+
allow: %w[allow/* /someting/dir/allow.html]
|
21
|
+
}
|
22
|
+
]
|
23
|
+
end
|
24
|
+
let(:expected) do
|
25
|
+
expected = <<~ROBOTS
|
26
|
+
User-Agent: GoogleBot
|
27
|
+
Disallow: /tmp/*
|
28
|
+
Disallow: /someting/dir/disallow.html
|
29
|
+
Allow: /allow/*
|
30
|
+
Allow: /someting/dir/allow.html
|
31
|
+
|
32
|
+
User-Agent: GoogleBot-Image
|
33
|
+
Disallow: /tmp/*
|
34
|
+
Disallow: /someting/dir/disallow.html
|
35
|
+
Allow: /allow/*
|
36
|
+
Allow: /someting/dir/allow.html
|
37
|
+
ROBOTS
|
38
|
+
expected.chomp
|
39
|
+
end
|
40
|
+
|
41
|
+
it { is_expected.to eq expected }
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'without option' do
|
45
|
+
let(:rules) { {} }
|
46
|
+
|
47
|
+
it { is_expected.to be_nil }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'middleman-robots/generators/sitemap_uri'
|
4
|
+
|
5
|
+
RSpec.describe Middleman::Robots::Generators::SitemapUri do
|
6
|
+
describe '#text' do
|
7
|
+
subject { described_class.new(uri).text }
|
8
|
+
|
9
|
+
context 'with uri value is' do
|
10
|
+
context 'URL String' do
|
11
|
+
let(:uri) { 'https://example.com/sitemap.xml' }
|
12
|
+
|
13
|
+
it { is_expected.to eq "Sitemap: #{uri}" }
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'nil' do
|
17
|
+
let(:uri) { nil }
|
18
|
+
|
19
|
+
it { is_expected.to be_nil }
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'empty' do
|
23
|
+
let(:uri) { '' }
|
24
|
+
|
25
|
+
it { is_expected.to be_nil }
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'ERROR when' do
|
29
|
+
subject { -> { described_class.new(rule).text } }
|
30
|
+
|
31
|
+
context 'Array' do
|
32
|
+
let(:rule) { { uri: %w[a b] } }
|
33
|
+
|
34
|
+
it { is_expected.to raise_error ArgumentError }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'Numeric' do
|
38
|
+
let(:rule) { { uri: 1_000 } }
|
39
|
+
|
40
|
+
it { is_expected.to raise_error ArgumentError }
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'Symbol' do
|
44
|
+
let(:rule) { { uri: :uri } }
|
45
|
+
|
46
|
+
it { is_expected.to raise_error ArgumentError }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
RSpec.configure do |config|
|
4
|
+
config.expect_with :rspec do |expectations|
|
5
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
6
|
+
end
|
7
|
+
|
8
|
+
config.mock_with :rspec do |mocks|
|
9
|
+
mocks.verify_partial_doubles = true
|
10
|
+
end
|
11
|
+
|
12
|
+
config.shared_context_metadata_behavior = :apply_to_host_groups
|
13
|
+
end
|
metadata
CHANGED
@@ -1,27 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: middleman-robots
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuya Matsushima
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: middleman
|
14
|
+
name: middleman-cli
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '4.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: middleman-core
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '4.0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
25
39
|
- !ruby/object:Gem::Version
|
26
40
|
version: '4.0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
@@ -94,6 +108,20 @@ dependencies:
|
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '12.3'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
97
125
|
- !ruby/object:Gem::Dependency
|
98
126
|
name: rubocop
|
99
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -116,7 +144,9 @@ executables: []
|
|
116
144
|
extensions: []
|
117
145
|
extra_rdoc_files: []
|
118
146
|
files:
|
147
|
+
- ".github/workflows/ci.yml"
|
119
148
|
- ".gitignore"
|
149
|
+
- ".rspec"
|
120
150
|
- ".rubocop.yml"
|
121
151
|
- ".travis.yml"
|
122
152
|
- Gemfile
|
@@ -124,6 +154,7 @@ files:
|
|
124
154
|
- README.jp.md
|
125
155
|
- README.md
|
126
156
|
- Rakefile
|
157
|
+
- cucumber.yml
|
127
158
|
- features/build.feature
|
128
159
|
- features/server.feature
|
129
160
|
- features/support/env.rb
|
@@ -135,18 +166,24 @@ files:
|
|
135
166
|
- lib/middleman-robots.rb
|
136
167
|
- lib/middleman-robots/extension.rb
|
137
168
|
- lib/middleman-robots/generator.rb
|
138
|
-
- lib/middleman-robots/
|
169
|
+
- lib/middleman-robots/generators/block.rb
|
170
|
+
- lib/middleman-robots/generators/blocks.rb
|
171
|
+
- lib/middleman-robots/generators/sitemap_uri.rb
|
139
172
|
- lib/middleman-robots/version.rb
|
140
173
|
- lib/middleman_extension.rb
|
141
174
|
- middleman-robots.gemspec
|
142
|
-
-
|
143
|
-
-
|
175
|
+
- spec/lib/middleman-robots/generator_spec.rb
|
176
|
+
- spec/lib/middleman-robots/generators/block_spec.rb
|
177
|
+
- spec/lib/middleman-robots/generators/blocks_spec.rb
|
178
|
+
- spec/lib/middleman-robots/generators/sitemap_uri_spec.rb
|
179
|
+
- spec/spec_helper.rb
|
144
180
|
- tmp/robots/.gitkeep
|
145
181
|
homepage: https://github.com/yterajima/middleman-robots
|
146
182
|
licenses:
|
147
183
|
- MIT
|
148
|
-
metadata:
|
149
|
-
|
184
|
+
metadata:
|
185
|
+
rubygems_mfa_required: 'true'
|
186
|
+
post_install_message:
|
150
187
|
rdoc_options: []
|
151
188
|
require_paths:
|
152
189
|
- lib
|
@@ -154,19 +191,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
154
191
|
requirements:
|
155
192
|
- - ">="
|
156
193
|
- !ruby/object:Gem::Version
|
157
|
-
version: 2.
|
194
|
+
version: 2.6.0
|
158
195
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
196
|
requirements:
|
160
197
|
- - ">="
|
161
198
|
- !ruby/object:Gem::Version
|
162
199
|
version: '0'
|
163
200
|
requirements: []
|
164
|
-
|
165
|
-
|
166
|
-
signing_key:
|
201
|
+
rubygems_version: 3.2.32
|
202
|
+
signing_key:
|
167
203
|
specification_version: 4
|
168
204
|
summary: Generate robots.txt by config.rb.
|
169
205
|
test_files:
|
170
206
|
- features/build.feature
|
171
207
|
- features/server.feature
|
172
208
|
- features/support/env.rb
|
209
|
+
- spec/lib/middleman-robots/generator_spec.rb
|
210
|
+
- spec/lib/middleman-robots/generators/block_spec.rb
|
211
|
+
- spec/lib/middleman-robots/generators/blocks_spec.rb
|
212
|
+
- spec/lib/middleman-robots/generators/sitemap_uri_spec.rb
|
213
|
+
- spec/spec_helper.rb
|
@@ -1,51 +0,0 @@
|
|
1
|
-
module Middleman
|
2
|
-
module Robots
|
3
|
-
# Robots Group Class
|
4
|
-
#
|
5
|
-
# Group class generate block in robots.txt
|
6
|
-
class Group
|
7
|
-
attr_reader :user_agent, :disallow, :allow
|
8
|
-
|
9
|
-
def initialize(rule)
|
10
|
-
@user_agent = generate_user_agent(rule)
|
11
|
-
@disallow = generate_disallow(rule)
|
12
|
-
@allow = generate_allow(rule)
|
13
|
-
end
|
14
|
-
|
15
|
-
def text
|
16
|
-
group = []
|
17
|
-
group << "User-Agent: #{@user_agent}" unless @user_agent.empty?
|
18
|
-
group << @disallow.collect { |item| "Disallow: #{item}" }.join("\n") if @disallow.length.positive?
|
19
|
-
group << @allow.collect { |item| "Allow: #{item}" }.join("\n") if @allow.length.positive?
|
20
|
-
group.join("\n") + "\n"
|
21
|
-
end
|
22
|
-
|
23
|
-
private
|
24
|
-
|
25
|
-
def generate_user_agent(rule)
|
26
|
-
return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
|
27
|
-
rule[:user_agent] || rule['user-agent']
|
28
|
-
end
|
29
|
-
|
30
|
-
def generate_disallow(rule)
|
31
|
-
paths = []
|
32
|
-
return paths unless rule.key?(:disallow)
|
33
|
-
|
34
|
-
rule[:disallow].each do |path|
|
35
|
-
paths << File.join('/', path)
|
36
|
-
end
|
37
|
-
paths
|
38
|
-
end
|
39
|
-
|
40
|
-
def generate_allow(rule)
|
41
|
-
paths = []
|
42
|
-
return paths unless rule.key?(:allow)
|
43
|
-
|
44
|
-
rule[:allow].each do |path|
|
45
|
-
paths << File.join('/', path)
|
46
|
-
end
|
47
|
-
paths
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
data/tests/test_generator.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'minitest/autorun'
|
2
|
-
require_relative '../lib/middleman-robots/generator.rb'
|
3
|
-
|
4
|
-
class TestGenerator < MiniTest::Test
|
5
|
-
def test_process
|
6
|
-
sitemap_uri = 'http://example.com/sitemap.xml'
|
7
|
-
generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
|
8
|
-
assert_equal expected, generator.process
|
9
|
-
end
|
10
|
-
|
11
|
-
private
|
12
|
-
|
13
|
-
def rules
|
14
|
-
[
|
15
|
-
{
|
16
|
-
user_agent: 'Googlebot',
|
17
|
-
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
18
|
-
allow: %w[allow/* /something/dir/file_allow.html]
|
19
|
-
},
|
20
|
-
{
|
21
|
-
user_agent: 'Googlebot-Image',
|
22
|
-
disallow: %w[tmp/* /something/dir/file_disallow.html],
|
23
|
-
allow: %w[allow/* /something/dir/file_allow.html]
|
24
|
-
}
|
25
|
-
]
|
26
|
-
end
|
27
|
-
|
28
|
-
def expected
|
29
|
-
"User-Agent: Googlebot
|
30
|
-
Disallow: /tmp/*
|
31
|
-
Disallow: /something/dir/file_disallow.html
|
32
|
-
Allow: /allow/*
|
33
|
-
Allow: /something/dir/file_allow.html
|
34
|
-
|
35
|
-
User-Agent: Googlebot-Image
|
36
|
-
Disallow: /tmp/*
|
37
|
-
Disallow: /something/dir/file_disallow.html
|
38
|
-
Allow: /allow/*
|
39
|
-
Allow: /something/dir/file_allow.html
|
40
|
-
|
41
|
-
Sitemap: http://example.com/sitemap.xml"
|
42
|
-
end
|
43
|
-
end
|
data/tests/test_group.rb
DELETED
@@ -1,120 +0,0 @@
|
|
1
|
-
require 'minitest/autorun'
|
2
|
-
require_relative '../lib/middleman-robots/group.rb'
|
3
|
-
|
4
|
-
class TestGroup < MiniTest::Test
|
5
|
-
def test_initialize
|
6
|
-
rule = {
|
7
|
-
user_agent: 'GoogleBot',
|
8
|
-
disallow: %w[tmp/* /someting/dir/disallow.html],
|
9
|
-
allow: %w[allow/* /someting/dir/allow.html]
|
10
|
-
}
|
11
|
-
group = Middleman::Robots::Group.new(rule)
|
12
|
-
|
13
|
-
assert_equal rule[:user_agent], group.user_agent
|
14
|
-
assert_equal %w[/tmp/* /someting/dir/disallow.html], group.disallow
|
15
|
-
assert_equal %w[/allow/* /someting/dir/allow.html], group.allow
|
16
|
-
end
|
17
|
-
|
18
|
-
def test_initialize_without_user_agent
|
19
|
-
rule = {
|
20
|
-
disallow: %w[/tmp/*],
|
21
|
-
allow: %w[/allow/*]
|
22
|
-
}
|
23
|
-
group = Middleman::Robots::Group.new(rule)
|
24
|
-
|
25
|
-
assert_equal '*', group.user_agent
|
26
|
-
assert_equal rule[:disallow], group.disallow
|
27
|
-
assert_equal rule[:allow], group.allow
|
28
|
-
end
|
29
|
-
|
30
|
-
def test_initialize_without_disallow
|
31
|
-
rule = {
|
32
|
-
user_agent: 'GoogleBot',
|
33
|
-
allow: %w[/allow/* /someting/dir/allow.html]
|
34
|
-
}
|
35
|
-
group = Middleman::Robots::Group.new(rule)
|
36
|
-
|
37
|
-
assert_equal rule[:user_agent], group.user_agent
|
38
|
-
assert_equal [], group.disallow
|
39
|
-
assert_equal rule[:allow], group.allow
|
40
|
-
end
|
41
|
-
|
42
|
-
def test_initialize_without_allow
|
43
|
-
rule = {
|
44
|
-
user_agent: 'GoogleBot',
|
45
|
-
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
46
|
-
}
|
47
|
-
group = Middleman::Robots::Group.new(rule)
|
48
|
-
|
49
|
-
assert_equal rule[:user_agent], group.user_agent
|
50
|
-
assert_equal rule[:disallow], group.disallow
|
51
|
-
assert_equal [], group.allow
|
52
|
-
end
|
53
|
-
|
54
|
-
def test_text
|
55
|
-
rule = {
|
56
|
-
user_agent: 'GoogleBot',
|
57
|
-
disallow: %w[tmp/* /someting/dir/disallow.html],
|
58
|
-
allow: %w[allow/* /someting/dir/allow.html]
|
59
|
-
}
|
60
|
-
group = Middleman::Robots::Group.new(rule)
|
61
|
-
|
62
|
-
expected = <<~ROBOTS
|
63
|
-
User-Agent: GoogleBot
|
64
|
-
Disallow: /tmp/*
|
65
|
-
Disallow: /someting/dir/disallow.html
|
66
|
-
Allow: /allow/*
|
67
|
-
Allow: /someting/dir/allow.html
|
68
|
-
ROBOTS
|
69
|
-
|
70
|
-
assert_equal expected, group.text
|
71
|
-
end
|
72
|
-
|
73
|
-
def test_text_without_user_agent
|
74
|
-
rule = {
|
75
|
-
disallow: %w[/tmp/*],
|
76
|
-
allow: %w[/allow/*]
|
77
|
-
}
|
78
|
-
group = Middleman::Robots::Group.new(rule)
|
79
|
-
|
80
|
-
expected = <<~ROBOTS
|
81
|
-
User-Agent: *
|
82
|
-
Disallow: /tmp/*
|
83
|
-
Allow: /allow/*
|
84
|
-
ROBOTS
|
85
|
-
|
86
|
-
assert_equal expected, group.text
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_text_without_disallow
|
90
|
-
rule = {
|
91
|
-
user_agent: 'GoogleBot',
|
92
|
-
allow: %w[/allow/* /someting/dir/allow.html]
|
93
|
-
}
|
94
|
-
group = Middleman::Robots::Group.new(rule)
|
95
|
-
|
96
|
-
expected = <<~ROBOTS
|
97
|
-
User-Agent: GoogleBot
|
98
|
-
Allow: /allow/*
|
99
|
-
Allow: /someting/dir/allow.html
|
100
|
-
ROBOTS
|
101
|
-
|
102
|
-
assert_equal expected, group.text
|
103
|
-
end
|
104
|
-
|
105
|
-
def test_text_without_allow
|
106
|
-
rule = {
|
107
|
-
user_agent: 'GoogleBot',
|
108
|
-
disallow: %w[/tmp/* /someting/dir/disallow.html]
|
109
|
-
}
|
110
|
-
group = Middleman::Robots::Group.new(rule)
|
111
|
-
|
112
|
-
expected = <<~ROBOTS
|
113
|
-
User-Agent: GoogleBot
|
114
|
-
Disallow: /tmp/*
|
115
|
-
Disallow: /someting/dir/disallow.html
|
116
|
-
ROBOTS
|
117
|
-
|
118
|
-
assert_equal expected, group.text
|
119
|
-
end
|
120
|
-
end
|