spider 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa130209d8ff28f2cfb7c42ec1915ca39b5a0895e568882134a59b67757c432d
-  data.tar.gz: 424d6d26fd66353515cbc7407fb0c5554a3566f19da0df526940c97025f4ef2c
+  metadata.gz: 393d7a240dbf77360ca4aeac52e8f162443903bac06817bbbf12440e597221f8
+  data.tar.gz: a26ff2a1f661fcde2712dc11f4a7fc0a9920e2c111bde01ef07b2135f0428c15
 SHA512:
-  metadata.gz: 686956d9960f6445e6f85dcf471b941154d232c7f9a0ff7c13008aed94f6a3a3a328e41abcbcbfa24bbedb5fe04bc481ec9ac31cbabf82806e5b7cb447075f79
-  data.tar.gz: 94ea96cb7b1cb71777d4b74a42d868d853b4adbb08185be2035d6658e1e027374ce3d4cc75109dfea49ba8fb95b44d7ae5948f8d4b03a1027c780eac4023e29f
+  metadata.gz: 72498a31555641b00d448d28cf7a20e652a2824c7891b9f0ca7a04c126d0481f3df98267c876e63f01d1103c56e6a747e448bb4968cdbab9245806dfdf25ca77
+  data.tar.gz: 141feb5b9446ad7f40c2c79ddbb1b43e70a93a6347115c5bbc96a2a55b47d0eef096b99f9586d3ca122d93d0e48e04b4d0b4f44b4316475053256f6645a6837c

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.8.0

data/lib/spider/included_in_file.rb

@@ -15,14 +15,14 @@ class IncludedInFile
   def initialize(filepath)
     @filepath = filepath
     # create file if not exists
-    File.write(@filepath, '') unless File.file?(@filepath)
+    File.write(@filepath, "") unless File.file?(@filepath)
     @urls = File.readlines(@filepath).map(&:chomp)
   end
 
   # Add an item to the file & array of URL.
   def <<(v)
     @urls << v.to_s
-    File.write(@filepath, "#{v}\r\n", File.size(@filepath), mode: 'a')
+    File.write(@filepath, "#{v}\r\n", File.size(@filepath), mode: "a")
   end
 
   # True if the item is in the file.

data/lib/spider/included_in_memcached.rb

@@ -1,6 +1,6 @@
 # Use memcached to track cycles.
 
-require 'dalli'
+require "dalli"
 
 # A specialized class using memcached to track items stored. It supports
 # three operations: new, <<, and include? . Together these can be used to

data/lib/spider/included_in_redis.rb

@@ -1,7 +1,7 @@
 # Use Redis to track cycles.
 
-require 'redis'
-require 'json'
+require "redis"
+require "json"
 
 # A specialized class using Redis to track items stored. It supports
 # three operations: new, <<, and include? . Together these can be used to

data/lib/spider/next_urls_in_sqs.rb

@@ -1,8 +1,8 @@
 # Use AmazonSQS to track nodes to visit.
 
-require 'rubygems'
-require 'right_aws'
-require 'yaml'
+require "rubygems"
+require "right_aws"
+require "yaml"
 
 # A specialized class using AmazonSQS to track nodes to walk. It supports
 # two operations: push and pop . Together these can be used to
@@ -20,7 +20,7 @@ class NextUrlsInSQS
   # Construct a new NextUrlsInSQS instance. All arguments here are
   # passed to RightAWS::SqsGen2 (part of the right_aws gem) or used
   # to set the AmazonSQS queue name (optional).
-  def initialize(aws_access_key, aws_secret_access_key, queue_name = 'ruby-spider')
+  def initialize(aws_access_key, aws_secret_access_key, queue_name = "ruby-spider")
     @sqs = RightAws::SqsGen2.new(aws_access_key, aws_secret_access_key)
     @queue = @sqs.queue(queue_name)
   end
@@ -30,14 +30,14 @@ class NextUrlsInSQS
   def pop
     while true
       message = @queue.pop
-      return YAML::load(message.to_s) unless message.nil?
+      return YAML.load(message.to_s) unless message.nil?
       sleep 5
     end
   end
 
   # Put data on the queue. Data is encoded with YAML.
   def push(a_msg)
-    encoded_message = YAML::dump(a_msg)
+    encoded_message = YAML.dump(a_msg)
     @queue.push(a_msg)
   end
 end

data/lib/spider/robot_rules.rb

@@ -11,13 +11,13 @@ require "uri"
 
 # Based on Perl's WWW::RobotRules module, by Gisle Aas.
 class RobotRules
-   def initialize( user_agent )
+   def initialize(user_agent)
      @user_agent = user_agent.scan(/\S+/).first.sub(%r{/.*},
 "").downcase
      @rules      = Hash.new { |rules, rule| rules[rule] = Array.new }
    end
 
-   def parse( text_uri, robots_data )
+   def parse(text_uri, robots_data)
      uri      = URI.parse(text_uri)
      location = "#{uri.host}:#{uri.port}"
      @rules.delete(location)
@@ -69,12 +69,12 @@ uri.scheme
      end
    end
 
-   def allowed?( text_uri )
+   def allowed?(text_uri)
      uri      = URI.parse(text_uri)
      location = "#{uri.host}:#{uri.port}"
      path     = uri.path
 
-     return true unless %w{http https}.include?(uri.scheme)
+     return true unless %w[http https].include?(uri.scheme)
 
      not @rules[location].any? { |rule| path.index(rule) == 0 }
    end

data/lib/spider/spider_instance.rb

@@ -1,41 +1,51 @@
 # Specialized spidering rules.
 
-require File.dirname(__FILE__)+'/robot_rules.rb'
-require 'open-uri'
-require 'uri'
-require 'net/http'
-require 'net/https'
+require File.dirname(__FILE__)+"/robot_rules.rb"
+require "open-uri"
+require "uri"
+require "net/http"
+require "net/https"
 
-module Net #:nodoc:
-  class HTTPResponse #:nodoc:
+module Net # :nodoc:
+  class HTTPResponse # :nodoc:
     def success?; false; end
     def redirect?; false; end
   end
-  class HTTPSuccess #:nodoc:
+  class HTTPSuccess # :nodoc:
     def success?; true; end
   end
-  class HTTPRedirection #:nodoc:
+  class HTTPRedirection # :nodoc:
     def redirect?; true; end
   end
 end
 
-class NilClass #:nodoc:
-  def merge(h); h; end
-end
-
 class SpiderInstance
-  def initialize(next_urls, seen = [], rules = nil, robots_seen = []) #:nodoc:
-    @url_checks  = []
-    @cache       = :memory
-    @callbacks   = {}
-    @next_urls   = [next_urls]
-    @seen        = seen
-    @rules       = rules || RobotRules.new("Ruby Spider #{Spider::VERSION}")
-    @robots_seen = robots_seen
-    @headers     = {}
-    @setup       = nil
-    @teardown    = nil
-    @interrupted = false
+  # Matches an anchor tag's attribute string carrying a rel value that should
+  # not be followed (nofollow, sponsored, or ugc), regardless of quoting.
+  NOFOLLOW_REL = /rel\s*=\s*["']?[^"'>]*\b(?:nofollow|sponsored|ugc)\b/i
+
+  # Seconds to wait before each HTTP request. Use to be polite to servers.
+  #  s.crawl_delay = 1
+  #
+  # Maximum number of pages to fetch before stopping. nil means no limit.
+  #  s.max_urls = 100
+  attr_writer :crawl_delay, :max_urls
+
+  def initialize(next_urls, seen = [], rules = nil, robots_seen = []) # :nodoc:
+    @url_checks   = []
+    @cache        = :memory
+    @callbacks    = {}
+    @next_urls    = [ next_urls ]
+    @seen         = seen
+    @rules        = rules || RobotRules.new("Ruby Spider #{Spider::VERSION}")
+    @robots_seen  = robots_seen
+    @headers      = {}
+    @setup        = nil
+    @teardown     = nil
+    @interrupted  = false
+    @crawl_delay  = 0
+    @max_urls     = nil
+    @urls_fetched = 0
   end
 
   # Add a predicate that determines whether to continue down this URL's path.
@@ -70,7 +80,7 @@ class SpiderInstance
     if cacher.respond_to?(:<<) && cacher.respond_to?(:include?)
       @seen = cacher
     else
-      raise ArgumentError, 'expected something that responds to << and included?'
+      raise ArgumentError, "expected something that responds to << and included?"
     end
   end
 
@@ -149,10 +159,10 @@ class SpiderInstance
     HeaderSetter.new(self)
   end
 
-  def raw_headers #:nodoc:
+  def raw_headers # :nodoc:
     @headers
   end
-  def raw_headers=(v) #:nodoc:
+  def raw_headers=(v) # :nodoc:
     @headers = v
   end
 
@@ -161,14 +171,14 @@ class SpiderInstance
     @headers = {}
   end
 
-  def start! #:nodoc:
+  def start! # :nodoc:
     trap("SIGINT") { @interrupted = true }
     begin
       next_urls = @next_urls.pop
       next_urls.each do |prior_url, urls|
-        urls = [urls] unless urls.kind_of?(Array)
+        urls = [ urls ] unless urls.kind_of?(Array)
         urls.map do |a_url|
-          [a_url, (URI.parse(a_url) rescue nil)]
+          [ a_url, (URI.parse(a_url) rescue nil) ]
         end.select do |a_url, parsed_url|
           allowable_url?(a_url, parsed_url)
         end.each do |a_url, parsed_url|
@@ -180,17 +190,18 @@ class SpiderInstance
             end
           end
           @teardown.call(a_url) unless @teardown.nil?
+          @interrupted = true if @max_urls && @urls_fetched >= @max_urls
           break if @interrupted
         end
       end
     end while !@next_urls.empty? && !@interrupted
   end
 
-  def stop! #:nodoc:
+  def stop! # :nodoc:
     @interrupted = true
   end
 
-  def success_or_failure(code) #:nodoc:
+  def success_or_failure(code) # :nodoc:
     if code > 199 && code < 300
       :success
     else
@@ -198,23 +209,23 @@ class SpiderInstance
     end
   end
 
-  def allowable_url?(a_url, parsed_url) #:nodoc:
+  def allowable_url?(a_url, parsed_url) # :nodoc:
     !parsed_url.nil? && !@seen.include?(parsed_url) && allowed?(a_url, parsed_url) &&
-      @url_checks.map{|url_check|url_check.call(a_url)}.all?
+      @url_checks.map { |url_check|url_check.call(a_url) }.all?
   end
 
   # True if the robots.txt for that URL allows access to it.
   def allowed?(a_url, parsed_url) # :nodoc:
-    return false unless ['http','https'].include?(parsed_url.scheme)
+    return false unless [ "http", "https" ].include?(parsed_url.scheme)
     u = "#{parsed_url.scheme}://#{parsed_url.host}:#{parsed_url.port}/robots.txt"
     parsed_u = URI.parse(u)
-    return false unless @url_checks.map{|url_check|url_check.call(a_url)}.all?
+    return false unless @url_checks.map { |url_check|url_check.call(a_url) }.all?
     begin
       unless @robots_seen.include?(u)
-        #open(u, 'User-Agent' => 'Ruby Spider',
+        # open(u, 'User-Agent' => 'Ruby Spider',
         #  'Accept' => 'text/html,text/xml,application/xml,text/plain', :ssl_verify => false) do |url|
         #  @rules.parse(u, url.read)
-        #end
+        # end
         get_page(parsed_u) do |r|
           @rules.parse(u, r.body)
         end
@@ -223,24 +234,26 @@ class SpiderInstance
       @rules.allowed?(a_url)
     rescue OpenURI::HTTPError
       true # No robots.txt
-    rescue Exception, Timeout::Error # to keep it from crashing
+    rescue StandardError, Timeout::Error # to keep it from crashing
       false
     end
   end
 
-  def get_page(parsed_url, &block) #:nodoc:
+  def get_page(parsed_url, &block) # :nodoc:
     @seen << parsed_url
     begin
+      sleep @crawl_delay if @crawl_delay && @crawl_delay > 0
       http = Net::HTTP.new(parsed_url.host, parsed_url.port)
-      if parsed_url.scheme == 'https'
+      if parsed_url.scheme == "https"
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
       # Uses start because http.finish cannot be called.
-      r = http.start {|h| h.request(Net::HTTP::Get.new(parsed_url.request_uri, @headers))}
+      r = http.start { |h| h.request(Net::HTTP::Get.new(parsed_url.request_uri, @headers)) }
       if r.redirect?
-        get_page(URI.parse(construct_complete_url(parsed_url,r['Location'])), &block)
+        get_page(URI.parse(construct_complete_url(parsed_url, r["Location"])), &block)
       else
+        @urls_fetched += 1
         block.call(r)
       end
     rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError => e
@@ -249,66 +262,62 @@ class SpiderInstance
     end
   end
 
-  def do_callbacks(a_url, resp, prior_url) #:nodoc:
-    cbs = [@callbacks[:every],
+  def do_callbacks(a_url, resp, prior_url) # :nodoc:
+    cbs = [ @callbacks[:every],
       resp.success? ?  @callbacks[:success] : @callbacks[:failure],
-      @callbacks[resp.code.to_i]]
+      @callbacks[resp.code.to_i] ]
 
     cbs.each do |cb|
       cb.call(a_url, resp, prior_url) if cb
     end
   end
 
-  def generate_next_urls(a_url, resp) #:nodoc:
+  def generate_next_urls(a_url, resp) # :nodoc:
+    # Only scan for links if the content-type is HTML or the URL ends with .html
+    content_type = resp["Content-Type"] || resp["content-type"] || ""
+    url_ends_with_html = a_url.downcase.end_with?(".html")
+
+    unless content_type.downcase.include?("text/html") || url_ends_with_html
+      return []
+    end
+
     web_page = resp.body
-    base_url = (web_page.scan(/base\s+href="(.*?)"/i).flatten +
-                [a_url[0,a_url.rindex('/')]])[0]
-    base_url = remove_trailing_slash(base_url)
-    web_page.scan(/href="(.*?)"/i).flatten.map do |link|
+    base_url = (web_page.scan(/base\s+href="(.*?)"/i).flatten + [ a_url ])[0]
+
+    # Extract each anchor tag once, then pull href and rel from the same tag,
+    # respecting rel="nofollow"/"sponsored"/"ugc".
+    web_page.scan(/<a\s([^>]*)>/i).flatten.map do |attrs|
+      next nil if attrs =~ NOFOLLOW_REL
+
+      href = attrs[/href\s*=\s*"([^"]*)"/i, 1]
+      next nil if href.nil? || href.empty?
+
       begin
-        parsed_link = URI.parse(link)
-        if parsed_link.fragment == '#'
+        parsed_link = URI.parse(href)
+        if parsed_link.fragment == "#"
           nil
         else
-          construct_complete_url(base_url, link, parsed_link)
+          construct_complete_url(base_url, href, parsed_link)
         end
-      rescue
+      rescue StandardError
         nil
       end
     end.compact
   end
 
-  def construct_complete_url(base_url, additional_url, parsed_additional_url = nil) #:nodoc:
+  def construct_complete_url(base_url, additional_url, parsed_additional_url = nil) # :nodoc:
     parsed_additional_url ||= URI.parse(additional_url)
-    case parsed_additional_url.scheme
-      when nil
-        u = base_url.is_a?(URI) ? base_url : URI.parse(base_url)
-        # Include port if it's not the default port
-        port_part = (u.port && ((u.scheme == 'http' && u.port != 80) || (u.scheme == 'https' && u.port != 443))) ? ":#{u.port}" : ""
-        if additional_url[0].chr == '/'
-          "#{u.scheme}://#{u.host}#{port_part}#{additional_url}"
-        elsif u.path.nil? || u.path == ''
-          "#{u.scheme}://#{u.host}#{port_part}/#{additional_url}"
-        elsif u.path[0].chr == '/'
-          "#{u.scheme}://#{u.host}#{port_part}#{u.path}/#{additional_url}"
-        else
-          "#{u.scheme}://#{u.host}#{port_part}/#{u.path}/#{additional_url}"
-        end
-    else
-      additional_url
-    end
-  end
-
-  def remove_trailing_slash(s) #:nodoc:
-    s.sub(%r{/*$},'')
+    return additional_url unless parsed_additional_url.scheme.nil?
+    base = base_url.is_a?(URI) ? base_url : URI.parse(base_url.to_s)
+    URI.join(base, additional_url).to_s
   end
 
-  class HeaderSetter #:nodoc:
+  class HeaderSetter # :nodoc:
     def initialize(si)
       @si = si
     end
-    def []=(k,v)
-      @si.raw_headers = @si.raw_headers.merge({k => v})
+    def []=(k, v)
+      @si.raw_headers = @si.raw_headers.merge({ k => v })
     end
   end
 end

data/lib/spider.rb

@@ -1,11 +1,10 @@
-require File.dirname(__FILE__)+'/spider/spider_instance'
+require File.dirname(__FILE__)+"/spider/spider_instance"
 
 # A spidering library for Ruby. Handles robots.txt, scraping, finding more
 # links, and doing it all over again.
 class Spider
-
   VERSION = File.read(
-    File.expand_path('../VERSION', __dir__)
+    File.expand_path("../VERSION", __dir__)
   ).strip.freeze
 
   def self.version
@@ -22,6 +21,12 @@ class Spider
   #      a_url =~ %r{^http://cashcats.biz.*}
   #    end
   #
+  #    # Be polite: wait one second between requests.
+  #    s.crawl_delay = 1
+  #
+  #    # Stop after fetching 100 pages.
+  #    s.max_urls = 100
+  #
   #    s.on 404 do |a_url, resp, prior_url|
   #      puts "URL not found: #{a_url}"
   #    end
@@ -37,7 +42,7 @@ class Spider
 
   def self.start_at(a_url, &block)
     rules    = RobotRules.new("Ruby Spider #{Spider::VERSION}")
-    a_spider = SpiderInstance.new({nil => [a_url]}, [], rules, [])
+    a_spider = SpiderInstance.new({ nil => [ a_url ] }, [], rules, [])
     block.call(a_spider)
     a_spider.start!
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spider
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - John Nagro
@@ -45,7 +45,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.14
 specification_version: 4
 summary: A Web spidering library
 test_files: []