husc 0.2.1 → 0.2.2

data/vendor/bundle/gems/webrobots-0.1.2/lib/webrobots/robotstxt.ry

@@ -1,487 +0,0 @@
-# -*- coding: utf-8 -*-
-
-class Parser
-
-rule
-  robotstxt		: opt_blanklines
-			  {
-			    @sitemaps = []
-			  }
-			  body
-			  {
-			    body = val[2]
-			    result = RobotsTxt.new(@site, body,
-			      :target => @target,
-                              :sitemaps => @sitemaps,
-                              :crawl_delay_handler => @crawl_delay_handler)
-			  }
-
-  body			:
-			| records
-			  opt_blanklines
-
-  opt_blanklines	:
-			| blanklines
-
-  blanklines		: blankline
-			| blanklines
-			  blankline
-
-  blankline		: EOL
-
-  opt_space		:
-			| SPACE
-
-  opt_commentlines	:
-			| commentlines
-
-  commentlines		: comment
-			| commentlines
-			  comment
-
-  comment		: opt_space COMMENT EOL
-			| 'sitemap' ':' opt_space VALUE eol_opt_comment
-			  {
-			    @sitemaps << val[3]
-			  }
-
-  records		: record
-			  {
-			    result = []
-			    result << val[0]
-			  }
-			| commentblock
-			  {
-			    result = []
-			  }
-			| records
-			  blanklines
-			  record
-			  {
-			    result << val[2]
-			  }
-			| records
-			  blanklines
-			  rulelines
-			  {
-			    val[2].each_with_index { |line, i|
-			      warn "%s line %d: %s: orphan rule line" %
-			        [@site.to_s, @rulelinenos[i], line.token] if $VERBOSE
-			    }
-			  }
-			| records
-			  blanklines
-			  commentblock
-
-  commentblock		: commentlines
-
-  record		: opt_commentlines
-			  agentlines
-			  opt_rulelines
-			  {
-			    result = Record.new(val[1], val[2])
-			  }
-
-  agentlines		: agentline
-			  {
-			    result = [val[0]]
-			  }
-			| agentlines
-			  agentline
-			  {
-			    result << val[1]
-			  }
-			| agentlines
-			  comment
-
-  agentline		: 'user-agent' ':' opt_space VALUE eol_opt_comment
-			  {
-			    result = AgentLine.new(val[0], val[3])
-			  }
-
-  opt_rulelines		:
-			| rulelines
-
-  rulelines		: ruleline
-			  {
-			    result = [result]
-			    @rulelinenos = []
-			  }
-			| rulelines
-			  ruleline
-			  {
-			    result << val[1]
-			    @rulelinenos << @lineno
-			  }
-			| rulelines
-			  comment
-
-  ruleline		: allowline
-			| disallowline
-			| crawldelayline
-			| extension
-
-  allowline		: 'allow' ':' opt_space VALUE eol_opt_comment
-			  {
-			    result = AllowLine.new(val[0], val[3])
-			  }
-
-  disallowline		: 'disallow' ':' opt_space VALUE eol_opt_comment
-			  {
-			    result = DisallowLine.new(val[0], val[3])
-			  }
-
-  crawldelayline	: 'crawl-delay' ':' opt_space VALUE eol_opt_comment
-			  {
-			    result = CrawlDelayLine.new(val[0], val[3])
-			  }
-
-  extension		: TOKEN ':' opt_space VALUE eol_opt_comment
-			  {
-			    result = ExtentionLine.new(val[0], val[3])
-			  }
-
-  eol_opt_comment	: EOL
-			| comment
-
----- header
-
-require 'strscan'
-
-class WebRobots
-  class Error < StandardError
-  end
-
-  class ParseError < Error
-    # The site's root URI
-    attr_reader :site
-
-    def initialize(message, site)
-      @message = message
-      @site = site
-    end
-
-    def to_s
-      @message
-    end
-  end
-
-  class RobotsTxt
----- inner
-
-      def initialize(target, crawl_delay_handler = nil)
-        super()
-        @target = target
-        @crawl_delay_handler = crawl_delay_handler
-      end
-
-      def parse!(input, site)
-        parse(input, site)
-      rescue Error => e
-        RobotsTxt.new(site, nil,
-          :error => e,
-          :target => @target,
-          :crawl_delay_handler => @crawl_delay_handler)
-      end
-
-      KNOWN_TOKENS = %w[User-agent Allow Disallow Crawl-delay Sitemap]
-      RE_KNOWN_TOKENS = /\A(#{KNOWN_TOKENS.map { |t| Regexp.quote(t) }.join('|')})\z/i
-
-      def parse(input, site)
-        @q ||= []
-        @errors = []
-        @lineno = 0
-        @site = site
-
-        string = input.respond_to?(:read) ? input.read : input
-        s = StringScanner.new(string)
-        value_expected = false
-
-        until s.eos?
-          @lineno += 1 if s.bol?
-          if t = s.scan(/[ \t]*(?:\r?\n|\z)/)
-            if value_expected
-              @q << [:VALUE, '']
-            end
-            @q << [:EOL, t]
-            value_expected = false
-          elsif t = s.scan(/[ \t]+/)
-            @q << [:SPACE, t]
-          elsif t = s.scan(/:/)
-            @q << [t, t]
-            value_expected = true
-          elsif t = s.scan(/#.*/)
-            if value_expected
-              @q << [:VALUE, '']
-            end
-            @q << [:COMMENT, t]
-          else
-            if value_expected
-              if t = s.scan(/.*?(?=[ \t]*(?:#|$))/)
-                @q << [:VALUE, t]
-              else
-                parse_error @lineno, "unexpected characters: %s" % s.check(/.*/)
-              end
-              value_expected = false
-            elsif t = s.scan(/[^\x00-\x1f\x7f()<>@,;:\\"\/\[\]?={}]+/)
-              case t
-              when RE_KNOWN_TOKENS
-                @q << [t.downcase, t]
-              else
-                @q << [:TOKEN, t]
-              end
-            else
-              parse_error "unexpected characters: %s" % s.check(/.*/)
-            end
-          end
-        end
-
-        @q << [:EOL, ''] if !@q.empty? && @q.last.first != :EOL
-
-        @pos = -1
-
-        do_parse
-      rescue Racc::ParseError => e
-        raise ParseError.new(e.message, @site)
-      ensure
-        @q.clear
-      end
-
-      def next_token
-        @q[@pos += 1]
-      end
-
-      def on_error(token_id, value, stack)
-        parse_error "unexpected %s: %s" % [token_to_str(token_id), value]
-      end
-
-      def parse_error(message)
-        message = "%s line %d: %s" % [@site.to_s, @lineno, message]
-        if @lax
-          @errors << message
-        else
-          raise Racc::ParseError, message
-        end
-      end
-
----- footer
-    def initialize(site, records, options = nil)
-      @timestamp = Time.now
-      @site = site
-      @options = options || {}
-      @last_checked_at = nil
-
-      @error = @options[:error]
-      @target = @options[:target]
-      @sitemaps = @options[:sitemaps] || []
-      @crawl_delay_handler = @options[:crawl_delay_handler]
-
-      if records && !records.empty?
-        @records, defaults = [], []
-        records.each { |record|
-          if record.default?
-            defaults << record
-          elsif !@target || record.match?(@target)
-            @records << record
-          end
-        }
-        @records.concat(defaults)
-      else
-        @records = []
-      end
-    end
-
-    attr_reader :timestamp, :site, :sitemaps
-    attr_accessor :error
-
-    def error!
-      raise @error if @error
-    end
-
-    def target(user_agent = nil)
-      if user_agent
-        raise ArgumentError, "this instance is targeted for #{@target}" if @target
-        user_agent
-      else
-        raise ArgumentError, "user_agent is mandatory for an untargeted instance" if !@target
-        @target
-      end
-    end
-    private :target
-
-    def find_record(user_agent = nil)
-      user_agent = target(user_agent)
-      @records.find { |record|
-        record.match?(user_agent)
-      }
-    end
-    private :find_record
-
-    def allow?(request_uri, user_agent = nil)
-      record = find_record(user_agent) or return true
-      allow = record.allow?(request_uri)
-      if delay = record.delay and @crawl_delay_handler
-        @crawl_delay_handler.call(delay, @last_checked_at)
-      end
-      @last_checked_at = Time.now
-      return allow
-    end
-
-    def crawl_delay(user_agent = nil)
-      record = find_record(user_agent) or return 0
-      record.delay or return 0
-    end
-
-    def options(user_agent = nil)
-      record = find_record(user_agent) or return {}
-      record.options
-    end
-
-    DISALLOW_ALL = <<-TXT
-User-Agent: *
-Disallow: /
-    TXT
-
-    def self.unfetchable(site, reason, target = nil)
-      Parser.new(target).parse(DISALLOW_ALL, site).tap { |robots_txt|
-        robots_txt.error = reason
-      }
-    end
-
-    class Record
-      def initialize(agentlines, rulelines)
-        @patterns = agentlines.map { |agentline| agentline.pattern }
-        @acls = []
-        @delay = nil
-        @options = {}
-        rulelines.each { |ruleline|
-          case ruleline
-          when AccessControlLine
-            @acls << ruleline
-          when CrawlDelayLine
-            @delay = ruleline.delay
-          else
-            @options[ruleline.token.downcase] = ruleline.value
-          end
-        } if rulelines
-        @acls.replace @acls.sort_by { |x|
-          [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
-        }
-      end
-
-      attr_reader :delay, :options
-
-      def match?(user_agent)
-        @patterns.any? { |pattern|
-          pattern.match(user_agent)
-        }
-      end
-
-      def default?
-        @patterns.include?(//)
-      end
-
-      def allow?(request_uri)
-        @acls.each { |acl|
-          if acl.match?(request_uri)
-            return acl.allow?
-          end
-        }
-        return true
-      end
-    end
-
-    class Line
-      def initialize(token, value)
-        @token = token
-        @value = value
-        compile
-      end
-
-      attr_reader :token, :value
-
-      def compile
-        self
-      end
-    end
-
-    class AgentLine < Line
-      def compile
-        if @value == '*'
-          @pattern = //
-        else
-          @pattern = Regexp.new(Regexp.quote(@value), Regexp::IGNORECASE)
-        end
-        self
-      end
-
-      attr_reader :pattern
-    end
-
-    class AccessControlLine < Line
-      def compile
-        @empty = @value.empty?
-        re_src = '\A'
-        s = StringScanner.new(@value)
-        until s.eos?
-          if t = s.scan(/[^%*$]+/)
-            re_src << Regexp.quote(t)
-          elsif t = s.scan(/%([0-9a-f]{2})/i)
-            c = s[1].to_i(16)
-            if c == 0x2f
-              re_src << '%2[fF]'
-            else
-              re_src << Regexp.quote('%c' % c)
-            end
-          elsif t = s.scan(/\*/)
-            re_src << '.*'
-          elsif t = s.scan(/\$/)
-            re_src << '\z'
-            break
-          else
-            re_src << Regexp.quote(s.scan(/./))
-          end
-        end
-        @pattern = Regexp.new(re_src, Regexp::MULTILINE)
-        self
-      end
-
-      def match?(request_uri)
-        return false if @empty
-        transformed = request_uri.gsub(/(%2[fF])|%([0-9a-f]{2})/i) { $1 || '%c' % $2.to_i(16) }
-        !!@pattern.match(transformed)
-      end
-    end
-
-    class AllowLine < AccessControlLine
-      def allow?
-        true
-      end
-    end
-
-    class DisallowLine < AccessControlLine
-      def allow?
-        false
-      end
-    end
-
-    class CrawlDelayLine < Line
-      def compile
-        case @value
-        when /\A((0|[1-9][0-9]*)\.[0-9]+)/
-          @delay = @value.to_f
-        when /\A(0|[1-9][0-9]*)/
-          @delay = @value.to_i
-        else
-          @delay = nil
-        end
-        self
-      end
-
-      attr_reader :delay
-    end
-
-    class ExtentionLine < Line
-    end
-  end
-end

data/vendor/bundle/gems/webrobots-0.1.2/lib/webrobots/version.rb

@@ -1,3 +0,0 @@
-module Webrobots
-  VERSION = "0.1.2"
-end

data/vendor/bundle/gems/webrobots-0.1.2/lib/webrobots.rb

@@ -1,213 +0,0 @@
-require 'webrobots/version'
-require 'webrobots/robotstxt'
-require 'uri'
-require 'net/https'
-require 'thread'
-if defined?(Nokogiri)
-  require 'webrobots/nokogiri'
-else
-  autoload :Nokogiri, 'webrobots/nokogiri'
-end
-
-class WebRobots
-  # Creates a WebRobots object for a robot named +user_agent+, with
-  # optional +options+.
-  #
-  # * :http_get => a custom method, proc, or anything that responds to
-  #   .call(uri), to be used for fetching robots.txt.  It must return
-  #   the response body if successful, return an empty string if the
-  #   resource is not found, and return nil or raise any error on
-  #   failure.  Redirects should be handled within this proc.
-  #
-  # * :crawl_delay => determines how to react to Crawl-delay
-  #   directives.  If +:sleep+ is given, WebRobots sleeps as demanded
-  #   when allowed?(url)/disallowed?(url) is called.  This is the
-  #   default behavior.  If +:ignore+ is given, WebRobots does
-  #   nothing.  If a custom method, proc, or anything that responds to
-  #   .call(delay, last_checked_at), it is called.
-  def initialize(user_agent, options = nil)
-    @user_agent = user_agent
-
-    options ||= {}
-    @http_get = options[:http_get] || method(:http_get)
-    crawl_delay_handler =
-      case value = options[:crawl_delay] || :sleep
-      when :ignore
-        nil
-      when :sleep
-        method(:crawl_delay_handler)
-      else
-        if value.respond_to?(:call)
-          value
-        else
-          raise ArgumentError, "invalid Crawl-delay handler: #{value.inspect}"
-        end
-      end
-
-    @parser = RobotsTxt::Parser.new(user_agent, crawl_delay_handler)
-    @parser_mutex = Mutex.new
-
-    @robotstxt = create_cache()
-  end
-
-  # :nodoc:
-  def create_cache
-    Hash.new	# Must respond to [], []=, delete and clear.
-  end
-
-  # Flushes robots.txt cache.
-  def flush_cache
-    @robotstxt.clear
-  end
-
-  # Returns the robot name initially given.
-  attr_reader :user_agent
-
-  # Tests if the robot is allowed to access a resource at +url+.  If a
-  # malformed URI string is given, URI::InvalidURIError is raised.  If
-  # a relative URI or a non-HTTP/HTTPS URI is given, ArgumentError is
-  # raised.
-  def allowed?(url)
-    site, request_uri = split_uri(url)
-    return true if request_uri == '/robots.txt'
-    robots_txt = get_robots_txt(site)
-    robots_txt.allow?(request_uri)
-  end
-
-  # Equivalent to !allowed?(url).
-  def disallowed?(url)
-    !allowed?(url)
-  end
-
-  # Returns the number of seconds that the configured agent should wait
-  # between successive requests to the site identified by +url+ according
-  # to the site's robots.txt +Crawl-delay+ directive.
-  def crawl_delay(url)
-    robots_txt_for(url).crawl_delay()
-  end
-
-  # Returns extended option values for a resource at +url+ in a hash
-  # with each field name lower-cased.  See allowed?() for a list of
-  # errors that may be raised.
-  def options(url)
-    robots_txt_for(url).options
-  end
-
-  # Equivalent to option(url)[token.downcase].
-  def option(url, token)
-    options(url)[token.downcase]
-  end
-
-  # Returns an array of Sitemap URLs.  See allowed?() for a list of
-  # errors that may be raised.
-  def sitemaps(url)
-    robots_txt_for(url).sitemaps
-  end
-
-  # Returns an error object if there is an error in fetching or
-  # parsing robots.txt of the site +url+.
-  def error(url)
-    robots_txt_for(url).error
-  end
-
-  # Raises the error if there was an error in fetching or parsing
-  # robots.txt of the site +url+.
-  def error!(url)
-    robots_txt_for(url).error!
-  end
-
-  # Removes robots.txt cache for the site +url+.
-  def reset(url)
-    site, = split_uri(url)
-    @robotstxt.delete(site)
-  end
-
-  private
-
-  def split_uri(url)
-    site =
-      if url.is_a?(URI)
-        url.dup
-      else
-        begin
-          URI.parse(url)
-        rescue => e
-          raise ArgumentError, e.message
-        end
-      end
-
-    site.scheme && site.host or
-      raise ArgumentError, "non-absolute URI: #{url}"
-
-    site.is_a?(URI::HTTP) or
-      raise ArgumentError, "non-HTTP/HTTPS URI: #{url}"
-
-    request_uri = site.request_uri
-    if (host = site.host).match(/[[:upper:]]/)
-      site.host = host.downcase
-    end
-    site.path = '/'
-    return site, request_uri
-  end
-
-  def robots_txt_for(url)
-    site, = split_uri(url)
-    get_robots_txt(site)
-  end
-
-  def get_robots_txt(site)
-    @robotstxt[site] ||= fetch_robots_txt(site)
-  end
-
-  def fetch_robots_txt(site)
-    begin
-      body = @http_get.call(site + 'robots.txt') or raise 'robots.txt unfetchable'
-    rescue => e
-      return RobotsTxt.unfetchable(site, e, @user_agent)
-    end
-    @parser_mutex.synchronize {
-      @parser.parse!(body, site)
-    }
-  end
-
-  def http_get(uri)
-    response = nil
-    referer = nil
-    5.times {
-      http = Net::HTTP.new(uri.host, uri.port)
-      if http.use_ssl = uri.is_a?(URI::HTTPS)
-        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        http.cert_store = OpenSSL::X509::Store.new.tap { |store|
-          store.set_default_paths
-        }
-      end
-      header = { 'User-Agent' => @user_agent }
-      header['Referer'] = referer if referer
-      # header is destroyed by this in ruby 1.9.2!
-      response = http.get(uri.request_uri, header)
-      case response
-      when Net::HTTPSuccess
-        return response.body
-      when Net::HTTPRedirection
-        referer = uri.to_s
-        uri = URI(response['location'])
-      when Net::HTTPClientError
-        return ''
-      end
-    }
-    case response
-    when Net::HTTPRedirection
-      # Treat too many redirections as not found
-      ''
-    else
-      raise "#{response.code} #{response.message}"
-    end
-  end
-
-  def crawl_delay_handler(delay, last_checked_at)
-    if last_checked_at
-      delay -= Time.now - last_checked_at
-      sleep delay if delay > 0
-    end
-  end
-end

data/vendor/bundle/gems/webrobots-0.1.2/test/helper.rb

@@ -1,25 +0,0 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-require 'shoulda'
-require 'webmock/test_unit'
-require 'vcr'
-
-VCR.configure do |c|
-  c.cassette_library_dir = 'test/vcr_cassettes'
-  c.hook_into :webmock
-end
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'webrobots'
-
-class Test::Unit::TestCase
-end