webrobots 0.0.13 → 0.1.0

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+
+coverage
+rdoc
+doc
+.yardoc
+
+/lib/webrobots/robotstxt.output

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - ree
+  - jruby-18mode
+  - jruby-19mode
+  - rbx-18mode
+  - rbx-19mode
+matrix:
+  allow_failures:
+    - rvm: rbx-18mode
+    - rvm: rbx-19mode

data/Gemfile

@@ -1,17 +1,4 @@
 source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
 
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
-group :development do
-  gem "racc", ">= 0"
-  gem "shoulda", ">= 0"
-  gem "bundler", ">= 1.0.0"
-  gem "jeweler", "~> 1.6.4"
-  gem "rcov", "~> 0.9.11"
-
-  # To test the webrobots/nokogiri module.
-  gem "nokogiri", ">= 1.4.4"
-end
+# Specify your gem's dependencies in webrobots.gemspec
+gemspec

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2011, 2012 Akinori MUSHA
+Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA
 
 All rights reserved.
 

data/README.rdoc

@@ -34,5 +34,5 @@ This is a library to help write robots.txt compliant web robots.
 
 == Copyright
 
-Copyright (c) 2010, 2011, 2012 Akinori MUSHA. See LICENSE.txt for
+Copyright (c) 2010, 2011, 2012, 2013 Akinori MUSHA. See LICENSE.txt for
 further details.

data/Rakefile

@@ -1,45 +1,22 @@
 # encoding: utf-8
 
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
+require 'bundler/gem_tasks'
 
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "webrobots"
-  gem.homepage = "https://github.com/knu/webrobots"
-  gem.license = "2-clause BSDL"
-  gem.summary = %Q{A Ruby library to help write robots.txt compliant web robots}
-  gem.description = <<-'EOS'
-This library helps write robots.txt compliant web robots in Ruby.
-  EOS
-  gem.email = "knu@idaemons.org"
-  gem.authors = ["Akinori MUSHA"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
+gemspec = Bundler::GemHelper.gemspec
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
+  test.libs << 'test'
+  test.test_files = gemspec.test_files
   test.verbose = true
 end
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "#{gemspec.name} #{gemspec.version}"
+  rdoc.rdoc_files.include(gemspec.extra_rdoc_files)
+  rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
 task :default => :test
@@ -49,13 +26,3 @@ task :test => 'lib/webrobots/robotstxt.rb'
 file 'lib/webrobots/robotstxt.rb' => 'lib/webrobots/robotstxt.ry' do
   sh 'racc', '-o', 'lib/webrobots/robotstxt.rb', 'lib/webrobots/robotstxt.ry'
 end
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "webrobots #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/lib/webrobots.rb

@@ -1,3 +1,4 @@
+require 'webrobots/version'
 require 'webrobots/robotstxt'
 require 'uri'
 require 'net/https'
@@ -17,13 +18,34 @@ class WebRobots
   #   the response body if successful, return an empty string if the
   #   resource is not found, and return nil or raise any error on
   #   failure.  Redirects should be handled within this proc.
+  #
+  # * :crawl_delay => determines how to react to Crawl-delay
+  #   directives.  If +:sleep+ is given, WebRobots sleeps as demanded
+  #   when allowed?(url)/disallowed?(url) is called.  This is the
+  #   default behavior.  If +:ignore+ is given, WebRobots does
+  #   nothing.  If a custom method, proc, or anything that responds to
+  #   .call(delay, last_checked_at), it is called.
   def initialize(user_agent, options = nil)
     @user_agent = user_agent
-    @parser = RobotsTxt::Parser.new(user_agent)
-    @parser_mutex = Mutex.new
 
     options ||= {}
     @http_get = options[:http_get] || method(:http_get)
+    crawl_delay_handler =
+      case value = options[:crawl_delay] || :sleep
+      when :ignore
+        nil
+      when :sleep
+        method(:crawl_delay_handler)
+      else
+        if value.respond_to?(:call)
+          value
+        else
+          raise ArgumentError, "invalid Crawl-delay handler: #{value.inspect}"
+        end
+      end
+
+    @parser = RobotsTxt::Parser.new(user_agent, crawl_delay_handler)
+    @parser_mutex = Mutex.new
 
     @robotstxt = create_cache()
   end
@@ -57,6 +79,13 @@ class WebRobots
     !allowed?(url)
   end
 
+  # Returns the number of seconds that the configured agent should wait
+  # between successive requests to the site identified by +url+ according
+  # to the site's robots.txt +Crawl-delay+ directive.
+  def crawl_delay(url)
+    robots_txt_for(url).crawl_delay()
+  end
+
   # Returns extended option values for a resource at +url+ in a hash
   # with each field name lower-cased.  See allowed?() for a list of
   # errors that may be raised.
@@ -169,4 +198,11 @@ class WebRobots
     }
     raise 'too many HTTP redirects'
   end
+
+  def crawl_delay_handler(delay, last_checked_at)
+    if last_checked_at
+      delay -= Time.now - last_checked_at
+      sleep delay if delay > 0
+    end
+  end
 end

data/lib/webrobots/robotstxt.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.7
+# This file is automatically generated by Racc 1.4.9
 # from Racc grammer file "".
 #
 
@@ -30,17 +30,21 @@ class WebRobots
   class RobotsTxt
 class Parser < Racc::Parser
 
-module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 169)
+module_eval(<<'...end robotstxt.ry/module_eval...', 'robotstxt.ry', 171)
 
-      def initialize(target = nil)
+      def initialize(target, crawl_delay_handler = nil)
         super()
         @target = target
+        @crawl_delay_handler = crawl_delay_handler
       end
 
       def parse!(input, site)
         parse(input, site)
       rescue Error => e
-        RobotsTxt.new(site, nil, :error => e, :target => @target)
+        RobotsTxt.new(site, nil,
+          :error => e,
+          :target => @target,
+          :crawl_delay_handler => @crawl_delay_handler)
       end
 
       KNOWN_TOKENS = %w[User-agent Allow Disallow Crawl-delay Sitemap]
@@ -334,7 +338,9 @@ module_eval(<<'.,.,', 'robotstxt.ry', 11)
   def _reduce_2(val, _values, result)
     			    body = val[2]
 			    result = RobotsTxt.new(@site, body,
-			      :target => @target, :sitemaps => @sitemaps)
+			      :target => @target,
+                              :sitemaps => @sitemaps,
+                              :crawl_delay_handler => @crawl_delay_handler)
 			  
     result
   end
@@ -368,7 +374,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 11)
 
 # reduce 16 omitted
 
-module_eval(<<'.,.,', 'robotstxt.ry', 42)
+module_eval(<<'.,.,', 'robotstxt.ry', 44)
   def _reduce_17(val, _values, result)
     			    @sitemaps << val[3]
 			  
@@ -376,7 +382,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 42)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 47)
+module_eval(<<'.,.,', 'robotstxt.ry', 49)
   def _reduce_18(val, _values, result)
     			    result = []
 			    result << val[0]
@@ -385,7 +391,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 47)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 52)
+module_eval(<<'.,.,', 'robotstxt.ry', 54)
   def _reduce_19(val, _values, result)
     			    result = []
 			  
@@ -393,7 +399,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 52)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 58)
+module_eval(<<'.,.,', 'robotstxt.ry', 60)
   def _reduce_20(val, _values, result)
     			    result << val[2]
 			  
@@ -401,7 +407,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 58)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 64)
+module_eval(<<'.,.,', 'robotstxt.ry', 66)
   def _reduce_21(val, _values, result)
     			    val[2].each_with_index { |line, i|
 			      warn "%s line %d: %s: orphan rule line" %
@@ -416,7 +422,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 64)
 
 # reduce 23 omitted
 
-module_eval(<<'.,.,', 'robotstxt.ry', 79)
+module_eval(<<'.,.,', 'robotstxt.ry', 81)
   def _reduce_24(val, _values, result)
     			    result = Record.new(val[1], val[2])
 			  
@@ -424,7 +430,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 79)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 84)
+module_eval(<<'.,.,', 'robotstxt.ry', 86)
   def _reduce_25(val, _values, result)
     			    result = [val[0]]
 			  
@@ -432,7 +438,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 84)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 89)
+module_eval(<<'.,.,', 'robotstxt.ry', 91)
   def _reduce_26(val, _values, result)
     			    result << val[1]
 			  
@@ -442,7 +448,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 89)
 
 # reduce 27 omitted
 
-module_eval(<<'.,.,', 'robotstxt.ry', 96)
+module_eval(<<'.,.,', 'robotstxt.ry', 98)
   def _reduce_28(val, _values, result)
     			    result = AgentLine.new(val[0], val[3])
 			  
@@ -454,7 +460,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 96)
 
 # reduce 30 omitted
 
-module_eval(<<'.,.,', 'robotstxt.ry', 104)
+module_eval(<<'.,.,', 'robotstxt.ry', 106)
   def _reduce_31(val, _values, result)
     			    result = [result]
 			    @rulelinenos = []
@@ -463,7 +469,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 104)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 110)
+module_eval(<<'.,.,', 'robotstxt.ry', 112)
   def _reduce_32(val, _values, result)
     			    result << val[1]
 			    @rulelinenos << @lineno
@@ -482,7 +488,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 110)
 
 # reduce 37 omitted
 
-module_eval(<<'.,.,', 'robotstxt.ry', 123)
+module_eval(<<'.,.,', 'robotstxt.ry', 125)
   def _reduce_38(val, _values, result)
     			    result = AllowLine.new(val[0], val[3])
 			  
@@ -490,7 +496,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 123)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 128)
+module_eval(<<'.,.,', 'robotstxt.ry', 130)
   def _reduce_39(val, _values, result)
     			    result = DisallowLine.new(val[0], val[3])
 			  
@@ -498,7 +504,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 128)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 133)
+module_eval(<<'.,.,', 'robotstxt.ry', 135)
   def _reduce_40(val, _values, result)
     			    result = CrawlDelayLine.new(val[0], val[3])
 			  
@@ -506,7 +512,7 @@ module_eval(<<'.,.,', 'robotstxt.ry', 133)
   end
 .,.,
 
-module_eval(<<'.,.,', 'robotstxt.ry', 138)
+module_eval(<<'.,.,', 'robotstxt.ry', 140)
   def _reduce_41(val, _values, result)
     			    result = ExtentionLine.new(val[0], val[3])
 			  
@@ -528,11 +534,12 @@ end   # class Parser
       @timestamp = Time.now
       @site = site
       @options = options || {}
-      @last_checked = nil
+      @last_checked_at = nil
 
       @error = @options[:error]
       @target = @options[:target]
       @sitemaps = @options[:sitemaps] || []
+      @crawl_delay_handler = @options[:crawl_delay_handler]
 
       if records && !records.empty?
         @records, defaults = [], []
@@ -578,14 +585,18 @@ end   # class Parser
     def allow?(request_uri, user_agent = nil)
       record = find_record(user_agent) or return true
       allow = record.allow?(request_uri)
-      if @last_checked and delay = record.delay
-        delay -= Time.now - @last_checked
-        sleep delay if delay > 0
+      if delay = record.delay and @crawl_delay_handler
+        @crawl_delay_handler.call(delay, @last_checked_at)
       end
-      @last_checked = Time.now
+      @last_checked_at = Time.now
       return allow
     end
 
+    def crawl_delay(user_agent = nil)
+      record = find_record(user_agent) or return 0
+      record.delay or return 0
+    end
+
     def options(user_agent = nil)
       record = find_record(user_agent) or return {}
       record.options

data/lib/webrobots/robotstxt.ry

@@ -11,7 +11,9 @@ rule
 			  {
 			    body = val[2]
 			    result = RobotsTxt.new(@site, body,
-			      :target => @target, :sitemaps => @sitemaps)
+			      :target => @target,
+                              :sitemaps => @sitemaps,
+                              :crawl_delay_handler => @crawl_delay_handler)
 			  }
 
   body			:
@@ -167,15 +169,19 @@ class WebRobots
   class RobotsTxt
 ---- inner
 
-      def initialize(target = nil)
+      def initialize(target, crawl_delay_handler = nil)
         super()
         @target = target
+        @crawl_delay_handler = crawl_delay_handler
       end
 
       def parse!(input, site)
         parse(input, site)
       rescue Error => e
-        RobotsTxt.new(site, nil, :error => e, :target => @target)
+        RobotsTxt.new(site, nil,
+          :error => e,
+          :target => @target,
+          :crawl_delay_handler => @crawl_delay_handler)
       end
 
       KNOWN_TOKENS = %w[User-agent Allow Disallow Crawl-delay Sitemap]
@@ -263,11 +269,12 @@ class WebRobots
       @timestamp = Time.now
       @site = site
       @options = options || {}
-      @last_checked = nil
+      @last_checked_at = nil
 
       @error = @options[:error]
       @target = @options[:target]
       @sitemaps = @options[:sitemaps] || []
+      @crawl_delay_handler = @options[:crawl_delay_handler]
 
       if records && !records.empty?
         @records, defaults = [], []
@@ -313,14 +320,18 @@ class WebRobots
     def allow?(request_uri, user_agent = nil)
       record = find_record(user_agent) or return true
       allow = record.allow?(request_uri)
-      if @last_checked and delay = record.delay
-        delay -= Time.now - @last_checked
-        sleep delay if delay > 0
+      if delay = record.delay and @crawl_delay_handler
+        @crawl_delay_handler.call(delay, @last_checked_at)
       end
-      @last_checked = Time.now
+      @last_checked_at = Time.now
       return allow
     end
 
+    def crawl_delay(user_agent = nil)
+      record = find_record(user_agent) or return 0
+      record.delay or return 0
+    end
+
     def options(user_agent = nil)
       record = find_record(user_agent) or return {}
       record.options

data/lib/webrobots/version.rb

@@ -0,0 +1,3 @@
+module Webrobots
+  VERSION = "0.1.0"
+end

data/test/test_webrobots.rb

@@ -384,6 +384,12 @@ Option1: Foo
 Option2: Hello
 Crawl-Delay: 1.5
 
+User-Agent: HerBot
+Disallow: /2heavy/
+Allow: /2heavy/*.html
+Option1: Baz
+Option2: Qux
+
 User-Agent: *
 Disallow: /2heavy/
 Allow: /2heavy/*.html
@@ -400,6 +406,9 @@ Option3: Hi
       }
 
       @robots_mybot = WebRobots.new('MyBot', :http_get => http_get)
+      @robots_mybot_ignore = WebRobots.new('MyBot', :http_get => http_get, :crawl_delay => :ignore)
+      @robots_mybot_custom = WebRobots.new('MyBot', :http_get => http_get, :crawl_delay => proc { |*args| @delay_args = args })
+      @robots_herbot = WebRobots.new('HerBot', :http_get => http_get)
       @robots_hisbot = WebRobots.new('HisBot', :http_get => http_get)
     end
 
@@ -411,6 +420,27 @@ Option3: Hi
       assert_equal 'Hello', @robots_mybot.option('http://www.example.org/', 'Option2')
       assert_equal 'Hello', options['option2']
 
+      options = @robots_mybot_ignore.options('http://www.example.org/')
+      assert_equal 2, options.size
+      assert_equal 'Foo',   @robots_mybot_ignore.option('http://www.example.org/', 'Option1')
+      assert_equal 'Foo',   options['option1']
+      assert_equal 'Hello', @robots_mybot_ignore.option('http://www.example.org/', 'Option2')
+      assert_equal 'Hello', options['option2']
+
+      options = @robots_mybot_custom.options('http://www.example.org/')
+      assert_equal 2, options.size
+      assert_equal 'Foo',   @robots_mybot_custom.option('http://www.example.org/', 'Option1')
+      assert_equal 'Foo',   options['option1']
+      assert_equal 'Hello', @robots_mybot_custom.option('http://www.example.org/', 'Option2')
+      assert_equal 'Hello', options['option2']
+
+      options = @robots_herbot.options('http://www.example.org/')
+      assert_equal 2, options.size
+      assert_equal 'Baz',   @robots_herbot.option('http://www.example.org/', 'Option1')
+      assert_equal 'Baz',   options['option1']
+      assert_equal 'Qux',    @robots_herbot.option('http://www.example.org/', 'Option2')
+      assert_equal 'Qux',    options['option2']
+
       options = @robots_hisbot.options('http://www.example.org/')
       assert_equal 2, options.size
       assert_equal 'Bar',   @robots_hisbot.option('http://www.example.org/', 'Option1')
@@ -422,11 +452,25 @@ Option3: Hi
         http://www.example.org/sitemap-host1.xml
         http://www.example.org/sitemap-host2.xml
       ], @robots_mybot.sitemaps('http://www.example.org/')
+      assert_equal %w[
+        http://www.example.org/sitemap-host1.xml
+        http://www.example.org/sitemap-host2.xml
+      ], @robots_mybot_ignore.sitemaps('http://www.example.org/')
+      assert_equal %w[
+        http://www.example.org/sitemap-host1.xml
+        http://www.example.org/sitemap-host2.xml
+      ], @robots_herbot.sitemaps('http://www.example.org/')
       assert_equal %w[
         http://www.example.org/sitemap-host1.xml
         http://www.example.org/sitemap-host2.xml
       ], @robots_hisbot.sitemaps('http://www.example.org/')
 
+      assert_equal 1.5, @robots_mybot.crawl_delay('http://www.example.org/')
+      assert_equal 1.5, @robots_mybot_ignore.crawl_delay('http://www.example.org/')
+      assert_equal 1.5, @robots_mybot_custom.crawl_delay('http://www.example.org/')
+      assert_equal 0, @robots_herbot.crawl_delay('http://www.example.org/')
+      assert_equal 0, @robots_hisbot.crawl_delay('http://www.example.org/')
+
       t1 = Time.now
       @robots_mybot.allowed?('http://www.example.org/')
       @robots_mybot.allowed?('http://www.example.org/article1.html')
@@ -435,6 +479,25 @@ Option3: Hi
       @robots_mybot.allowed?('http://www.example.org/article2.html')
       t3 = Time.now
       assert_in_delta 1.5, t3 - t2, 0.1
+
+      t1 = Time.now
+      @robots_mybot_ignore.allowed?('http://www.example.org/')
+      @robots_mybot_ignore.allowed?('http://www.example.org/article1.html')
+      t2 = Time.now
+      assert_in_delta 0, t2 - t1, 0.1
+      @robots_mybot_ignore.allowed?('http://www.example.org/article2.html')
+      t3 = Time.now
+      assert_in_delta 0, t3 - t2, 0.1
+
+      t1 = Time.now
+      @robots_mybot_custom.allowed?('http://www.example.org/')
+      @robots_mybot_custom.allowed?('http://www.example.org/article1.html')
+      t2 = Time.now
+      assert_in_delta 0, t2 - t1, 0.1
+      assert_instance_of Array, @delay_args
+      assert_equal 2, @delay_args.size
+      assert_equal 1.5, @delay_args[0]
+      assert_instance_of Time, @delay_args[1]
     end
   end
 

data/webrobots.gemspec

@@ -1,68 +1,33 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "webrobots/version"
 
 Gem::Specification.new do |s|
-  s.name = "webrobots"
-  s.version = "0.0.13"
+  s.name        = "webrobots"
+  s.version     = Webrobots::VERSION
+  s.authors     = ["Akinori MUSHA"]
+  s.email       = ["knu@idaemons.org"]
+  s.homepage    = %q{https://github.com/knu/webrobots}
+  s.licenses    = [%q{2-clause BSDL}]
+  s.summary     = %q{A Ruby library to help write robots.txt compliant web robots}
+  s.description = <<-'EOS'
+This library helps write robots.txt compliant web robots in Ruby.
+  EOS
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = s.files.grep(%r{/test_[^/]+\.rb$})
+  s.executables   = s.files.grep(%r{^bin/[^.]}).map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Akinori MUSHA"]
-  s.date = "2012-01-24"
-  s.description = "This library helps write robots.txt compliant web robots in Ruby.\n"
-  s.email = "knu@idaemons.org"
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
   ]
-  s.files = [
-    ".document",
-    "Gemfile",
-    "Gemfile.lock",
-    "LICENSE.txt",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "lib/webrobots.rb",
-    "lib/webrobots/nokogiri.rb",
-    "lib/webrobots/robotstxt.rb",
-    "lib/webrobots/robotstxt.ry",
-    "test/helper.rb",
-    "test/test_webrobots.rb",
-    "webrobots.gemspec"
-  ]
-  s.homepage = "https://github.com/knu/webrobots"
-  s.licenses = ["2-clause BSDL"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
-  s.summary = "A Ruby library to help write robots.txt compliant web robots"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<racc>, [">= 0"])
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, ["~> 0.9.11"])
-      s.add_development_dependency(%q<nokogiri>, [">= 1.4.4"])
-    else
-      s.add_dependency(%q<racc>, [">= 0"])
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, [">= 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, ["~> 0.9.11"])
-      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
-    end
-  else
-    s.add_dependency(%q<racc>, [">= 0"])
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, [">= 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, ["~> 0.9.11"])
-    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
-  end
+  s.add_development_dependency("rake", [">= 0.9.2.2"])
+  s.add_development_dependency("racc", [">= 0"]) unless RUBY_PLATFORM == "java"
+  s.add_development_dependency("shoulda", [">= 0"])
+  s.add_development_dependency("rdoc", ["> 2.4.2"])
+  s.add_development_dependency("bundler", [">= 1.2"])
+  s.add_development_dependency("nokogiri", [">= 1.4.4"])
 end
-

metadata

@@ -1,172 +1,162 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: webrobots
-version: !ruby/object:Gem::Version 
-  hash: 5
+version: !ruby/object:Gem::Version
+  version: 0.1.0
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 13
-  version: 0.0.13
 platform: ruby
-authors: 
+authors:
 - Akinori MUSHA
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-01-24 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
+date: 2013-02-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id001
-  name: racc
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id002
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 2.4.2
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  version_requirements: *id003
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+- !ruby/object:Gem::Dependency
   name: bundler
-  prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 6
-        - 4
-        version: 1.6.4
-  version_requirements: *id004
-  name: jeweler
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
   type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 45
-        segments: 
-        - 0
-        - 9
-        - 11
-        version: 0.9.11
-  version_requirements: *id005
-  name: rcov
   prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency 
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 4
-        - 4
-        version: 1.4.4
-  version_requirements: *id006
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
   name: nokogiri
-  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.4
   type: :development
-description: |
-  This library helps write robots.txt compliant web robots in Ruby.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.4
+description: ! 'This library helps write robots.txt compliant web robots in Ruby.
 
-email: knu@idaemons.org
+'
+email:
+- knu@idaemons.org
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
-files: 
+files:
 - .document
+- .gitignore
+- .travis.yml
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.rdoc
 - Rakefile
-- VERSION
 - lib/webrobots.rb
 - lib/webrobots/nokogiri.rb
 - lib/webrobots/robotstxt.rb
 - lib/webrobots/robotstxt.ry
+- lib/webrobots/version.rb
 - test/helper.rb
 - test/test_webrobots.rb
 - webrobots.gemspec
 homepage: https://github.com/knu/webrobots
-licenses: 
+licenses:
 - 2-clause BSDL
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: A Ruby library to help write robots.txt compliant web robots
-test_files: []
-
+test_files:
+- test/test_webrobots.rb

data/Gemfile.lock

@@ -1,24 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    git (1.2.5)
-    jeweler (1.6.4)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
-    nokogiri (1.5.0)
-    racc (1.4.7)
-    rake (0.9.2.2)
-    rcov (0.9.11)
-    shoulda (2.11.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (>= 1.0.0)
-  jeweler (~> 1.6.4)
-  nokogiri (>= 1.4.4)
-  racc
-  rcov (~> 0.9.11)
-  shoulda

data/VERSION

@@ -1 +0,0 @@
-0.0.13