middleman-robots 1.2.3 → 1.3.0

data/lib/middleman-robots/extension.rb

@@ -1,69 +1,31 @@
+require 'middleman-robots/generator'
+
 module Middleman
   module Robots
     # Robots Extension Class
-    #
-    # Create robots.txt when `$ middleman build`
     class Extension < ::Middleman::Extension
       option :rules, [], 'List of rules about sitemap.xml'
       option :sitemap, false, 'URI of sitemap.xml'
 
       def initialize(app, options_hash = {}, &block)
         super
-        build_dir = app.config.build_dir
-
-        data = rules(options.rules) + sitemap(options.sitemap)
-        data.gsub!(/\n+$/, "\n")
-
-        app.after_build do
-          File.open(File.join(build_dir, 'robots.txt'), 'w') do |file|
-            file.puts(data)
-          end
-          logger.info '== middleman-robots: robots.txt created =='
-        end
-      end
-
-      def rules(rules)
-        return '' if rules.empty?
-        data = []
-        rules.each do |rule|
-          row = []
-          row << user_agent(rule)
-          row << disallow(rule)
-          row << allow(rule)
-          row.compact!
-          data << row.join("\n") + "\n\n" if row.length > 0
-        end
-        data.join('')
-      end
-
-      def user_agent(rule)
-        return unless rule.key?('user-agent') || rule.key?(:user_agent)
-        user_agent = rule[:user_agent] || rule['user-agent']
-        "User-Agent: #{user_agent}"
-      end
-
-      def disallow(rule)
-        return unless rule.key?(:disallow)
-        lines = []
-        rule[:disallow].each do |path|
-          path = File.join('/', path) unless /^\// =~ path
-          lines << "Disallow: #{path}"
-        end
-        lines
       end
 
-      def allow(rule)
-        return unless rule.key?(:allow)
-        lines = []
-        rule[:allow].each do |path|
-          path = File.join('/' + path) unless /^\// =~ path
-          lines << "Allow: #{path}"
-        end
-        lines
+      def manipulate_resource_list(resources)
+        data   = Generator.new(options.rules, options.sitemap).process
+        robots = Middleman::Sitemap::Resource.new(
+          app.sitemap,
+          'robots.txt',
+          File.join(template_dir, 'robots.txt.erb')
+        )
+        robots.add_metadata(locals: {robots: data})
+
+        logger.info '== middleman-robots: robots.txt added to resources =='
+        resources << robots
       end
 
-      def sitemap(path)
-        path ? "Sitemap: #{path}" : ''
+      def template_dir
+        File.expand_path(File.join('..', 'templates'), __FILE__)
       end
     end
   end

data/lib/middleman-robots/generator.rb

@@ -0,0 +1,42 @@
+require 'middleman-robots/group'
+
+module Middleman
+  module Robots
+    # Robots Text Generator Class
+    class Generator
+      def initialize(rules, sitemap_uri)
+        @rules = rules
+        @sitemap_uri = sitemap_uri
+      end
+
+      def process
+        blocks  = block_text
+        sitemap = sitemap_text
+
+        if !blocks.empty? && !sitemap.empty?
+          blocks + "\n" + sitemap
+        elsif !blocks.empty?
+          blocks
+        elsif !sitemap.empty?
+          sitemap
+        else
+          ''
+        end
+      end
+
+      private
+        def block_text
+          return '' if @rules.empty?
+          data = []
+          @rules.each do |rule|
+            data << Group.new(rule).text
+          end
+          data.join("\n")
+        end
+
+        def sitemap_text
+          @sitemap_uri ? "Sitemap: #{@sitemap_uri}" : ''
+        end
+    end
+  end
+end

data/lib/middleman-robots/group.rb

@@ -0,0 +1,50 @@
+module Middleman
+  module Robots
+    # Robots Group Class
+    #
+    # Group class generate block in robots.txt
+    class Group
+      attr_reader :user_agent, :disallow, :allow
+
+      def initialize(rule)
+        @user_agent = generate_user_agent(rule)
+        @disallow   = generate_disallow(rule)
+        @allow      = generate_allow(rule)
+      end
+
+      def text
+        group = []
+        group << "User-Agent: #{@user_agent}" unless @user_agent.empty?
+        group << @disallow.collect{|item| "Disallow: #{item}" }.join("\n") if @disallow.length > 0
+        group << @allow.collect{|item| "Allow: #{item}" }.join("\n") if @allow.length > 0
+        group.join("\n") + "\n"
+      end
+
+      private
+        def generate_user_agent(rule)
+          return '*' unless rule.key?('user-agent') || rule.key?(:user_agent)
+          rule[:user_agent] || rule['user-agent']
+        end
+
+        def generate_disallow(rule)
+          paths = []
+          return paths unless rule.has_key?(:disallow)
+
+          rule[:disallow].each do |path|
+            paths << File.join('/', path)
+          end
+          paths
+        end
+
+        def generate_allow(rule)
+          paths = []
+          return paths unless rule.has_key?(:allow)
+
+          rule[:allow].each do |path|
+            paths << File.join('/', path)
+          end
+          paths
+        end
+    end
+  end
+end

data/lib/middleman-robots/templates/robots.txt.erb

@@ -0,0 +1 @@
+<%= robots %>

data/lib/middleman-robots/version.rb

@@ -1,6 +1,6 @@
 module Middleman
   # Middleman robots module
   module Robots
-    VERSION = '1.2.3'
+    VERSION = '1.3.0'
   end
 end

data/middleman-robots.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = Middleman::Robots::VERSION
   spec.authors       = ["Yuya Matsushima"]
   spec.email         = ["terra@e2esound.com"]
-  spec.summary       = %q{Create robots.txt when do 'build'.}
-  spec.description   = %q{Create robots.txt when do 'build'.}
+  spec.summary       = %q{Generate robots.txt by config.rb.}
+  spec.description   = %q{Generate robots.txt by config.rb.}
   spec.homepage      = "https://github.com/yterajima/middleman-robots"
   spec.license       = "MIT"
 

data/tests/test_generator.rb

@@ -0,0 +1,37 @@
+require 'minitest/autorun'
+require_relative '../lib/middleman-robots/generator.rb'
+
+class Test_Generator < MiniTest::Test
+  def test_process
+    rules = [
+      {
+        user_agent: 'Googlebot',
+        disallow:  %w(tmp/* /something/dir/file_disallow.html),
+        allow:  %w(allow/* /something/dir/file_allow.html)
+      },
+      {
+        user_agent: 'Googlebot-Image',
+        disallow:  %w(tmp/* /something/dir/file_disallow.html),
+        allow:  %w(allow/* /something/dir/file_allow.html)
+      }
+    ]
+    sitemap_uri = 'http://example.com/sitemap.xml'
+    generator = Middleman::Robots::Generator.new(rules, sitemap_uri)
+
+    expected = "User-Agent: Googlebot
+Disallow: /tmp/*
+Disallow: /something/dir/file_disallow.html
+Allow: /allow/*
+Allow: /something/dir/file_allow.html
+
+User-Agent: Googlebot-Image
+Disallow: /tmp/*
+Disallow: /something/dir/file_disallow.html
+Allow: /allow/*
+Allow: /something/dir/file_allow.html
+
+Sitemap: http://example.com/sitemap.xml"
+
+    assert_equal expected, generator.process
+  end
+end

data/tests/test_group.rb

@@ -0,0 +1,121 @@
+require 'minitest/autorun'
+require_relative '../lib/middleman-robots/group.rb'
+
+class Test_Group < MiniTest::Test
+
+  def test_initialize
+    rule = {
+      user_agent: 'GoogleBot',
+      disallow: %w(tmp/* /someting/dir/disallow.html),
+      allow: %w(allow/* /someting/dir/allow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    assert_equal rule[:user_agent], group.user_agent
+    assert_equal %w(/tmp/* /someting/dir/disallow.html), group.disallow
+    assert_equal %w(/allow/* /someting/dir/allow.html), group.allow
+  end
+
+  def test_initialize_without_user_agent
+    rule = {
+      disallow: %w(/tmp/*),
+      allow: %w(/allow/*),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    assert_equal '*', group.user_agent
+    assert_equal rule[:disallow], group.disallow
+    assert_equal rule[:allow], group.allow
+  end
+
+  def test_initialize_without_disallow
+    rule = {
+      user_agent: 'GoogleBot',
+      allow: %w(/allow/* /someting/dir/allow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    assert_equal rule[:user_agent], group.user_agent
+    assert_equal [], group.disallow
+    assert_equal rule[:allow], group.allow
+  end
+
+  def test_initialize_without_allow
+    rule = {
+      user_agent: 'GoogleBot',
+      disallow: %w(/tmp/* /someting/dir/disallow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    assert_equal rule[:user_agent], group.user_agent
+    assert_equal rule[:disallow], group.disallow
+    assert_equal [], group.allow
+  end
+
+  def test_text
+    rule = {
+      user_agent: 'GoogleBot',
+      disallow: %w(tmp/* /someting/dir/disallow.html),
+      allow: %w(allow/* /someting/dir/allow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    expected = <<-END
+User-Agent: GoogleBot
+Disallow: /tmp/*
+Disallow: /someting/dir/disallow.html
+Allow: /allow/*
+Allow: /someting/dir/allow.html
+    END
+
+    assert_equal expected, group.text
+  end
+
+  def test_text_without_user_agent
+    rule = {
+      disallow: %w(/tmp/*),
+      allow: %w(/allow/*),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    expected = <<-END
+User-Agent: *
+Disallow: /tmp/*
+Allow: /allow/*
+    END
+
+    assert_equal expected, group.text
+  end
+
+  def test_text_without_disallow
+    rule = {
+      user_agent: 'GoogleBot',
+      allow: %w(/allow/* /someting/dir/allow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    expected = <<-END
+User-Agent: GoogleBot
+Allow: /allow/*
+Allow: /someting/dir/allow.html
+    END
+
+    assert_equal expected, group.text
+  end
+
+  def test_text_without_allow
+    rule = {
+      user_agent: 'GoogleBot',
+      disallow: %w(/tmp/* /someting/dir/disallow.html),
+    }
+    group = Middleman::Robots::Group.new(rule)
+
+    expected = <<-END
+User-Agent: GoogleBot
+Disallow: /tmp/*
+Disallow: /someting/dir/disallow.html
+    END
+
+    assert_equal expected, group.text
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: middleman-robots
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.3.0
 platform: ruby
 authors:
 - Yuya Matsushima
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-03 00:00:00.000000000 Z
+date: 2017-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: middleman
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10'
-description: Create robots.txt when do 'build'.
+description: Generate robots.txt by config.rb.
 email:
 - terra@e2esound.com
 executables: []
@@ -108,17 +108,28 @@ files:
 - README.jp.md
 - README.md
 - Rakefile
-- features/robots.feature
+- features/build.feature
+- features/server.feature
 - features/support/env.rb
 - fixtures/basic-app/source/index.html.erb
 - fixtures/basic-app/source/layouts/layout.erb
 - fixtures/basic-app/source/stylesheets/all.css
 - fixtures/basic-app/source/stylesheets/normalize.css
+- fixtures/server-app/config.rb
+- fixtures/server-app/source/index.html.erb
+- fixtures/server-app/source/layouts/layout.erb
+- fixtures/server-app/source/stylesheets/all.css
+- fixtures/server-app/source/stylesheets/normalize.css
 - lib/middleman-robots.rb
 - lib/middleman-robots/extension.rb
+- lib/middleman-robots/generator.rb
+- lib/middleman-robots/group.rb
+- lib/middleman-robots/templates/robots.txt.erb
 - lib/middleman-robots/version.rb
 - lib/middleman_extension.rb
 - middleman-robots.gemspec
+- tests/test_generator.rb
+- tests/test_group.rb
 homepage: https://github.com/yterajima/middleman-robots
 licenses:
 - MIT
@@ -139,10 +150,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Create robots.txt when do 'build'.
+summary: Generate robots.txt by config.rb.
 test_files:
-- features/robots.feature
+- features/build.feature
+- features/server.feature
 - features/support/env.rb