site_mapper 0.0.10 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49d4c0ab215ac8872234e3c552275f89688f997d
-  data.tar.gz: 4b75bae288524a38fe3a8f7d6df9ab42e715211a
+  metadata.gz: dc4b21c14dc15f1cc6df4b6406d12acf5cb821d9
+  data.tar.gz: 34ef0ab2fcd0a74bbcdd53d9e47681d6440f951d
 SHA512:
-  metadata.gz: e0cad8aedfb27fc5a4d56b297098b76a9abd43df86b6c20b9d6f228256091191bb020e620217a9bb40bdc68c981bf8d2ca4fdf2930c55158bb321fa11dfc960b
-  data.tar.gz: c3b1c1863e3de70793d127772b7a314b1e788483fe9151e441c1c209b7fed59c8b210cdae5d453017b0d49eff179f7b51eee47544f68a15c6a62c94783b7b24e
+  metadata.gz: 1d4da1f2753dfb5f06ea577c02183efbf4cb919b783ce128f07b46f29b6af7a330cc01d839895ea4d4fb53cf68db2a6b5adccba47530a0278aaca9bfe1fa4c02
+  data.tar.gz: c32dde9478240d63b63d6e521e04f3c914322b544c79eee7ee25e1b2ea46b5ad2529ddc7d778d30fe2c10cde074f88a72c53458a814fc5d2ab74bc87288e63ed

data/lib/site_mapper.rb

@@ -20,10 +20,19 @@ module SiteMapper
   # @param [String] link to domain
   # @param [Hash] options hash
   # @example Collect all URLs from example.com
-  #   SiteMapper.map('example.com')
+  #    SiteMapper.map('example.com')
+  # @example Collect all URLs from example.com with custom User-agent
+  #    SiteMapper.map('example.com', user_agent: 'MyUserAgent')
+  # @example Collect all URLs from example.com with custom logger class
+  #    class MyLogger
+  #      def self.log(msg);     puts msg;end
+  #      def self.err_log(msg); puts msg;end
+  #    end
+  #    SiteMapper.map('example.com', logger: MyLogger)
   def self.map(link, options = {})
-    set_logger(options[:logger])
-    Crawler.collect_urls(link) { |url| yield(url) if block_given? }
+    set_logger(options.delete(:logger))
+    options = { user_agent: USER_AGENT }.merge(options)
+    Crawler.collect_urls(link, options) { |url| yield(url) if block_given? }
   end
 
   # Set logger.

data/lib/site_mapper/crawl_url.rb

@@ -1,85 +1,50 @@
 module SiteMapper
   # Crawl URL formatter.
   class CrawlUrl
-    attr_reader :resolved_base_url, :base_hostname
+    attr_reader :resolved_base_url
 
+    # Too many request error message
+    TOO_MANY_REQUEST_MSG = "You're being challenged with a 'too many requests' captcha"
+
+    # Initialize CrawlUrl
     # @param [String] base_url
+    # @example Intitialize CrawlUrl with example.com as base_url
+    #   CrawlUrl.new('example.com')
     def initialize(base_url)
-      @resolved_base_url = Request.resolve_url(base_url, with_query: false)
+      @resolved_base_url = Request.resolve_url(base_url) # "#{protocol}#{host}"
       @base_hostname     = URI.parse(@resolved_base_url).hostname
-      @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
     end
 
     # Given a link it constructs the absolute path,
     # if valid URL & URL has same domain as @resolved_base_url.
-    # @param [String] raw_url url found on page
-    # @param [String] get_url current page url
+    # @param [String] page_url url found on page
+    # @param [String] current_url current page url
     # @return [String] with absolute path to resource
     # @example Construct absolute URL for '/path', example.com
     #   cu = CrawlUrl.new('example.com')
     #   cu.absolute_url_from('/path', 'example.com/some/path')
     #   # => http://example.com/some/path
-    def absolute_url_from(raw_url, get_url)
-      return nil unless eligible_url?(raw_url)
-      parsed_url = URI.parse(raw_url) rescue URI.parse('')  
-      if parsed_url.relative?
-        url_from_relative(raw_url, get_url)
-      elsif same_domain?(raw_url, @resolved_base_url)
-        raw_url
-      else
-        nil
-      end
+    def absolute_url_from(page_url, current_url)
+      return unless eligible_url?(page_url)
+      parsed_uri = URI.join(current_url, page_url) rescue return
+      return unless parsed_uri.hostname == @base_hostname
+      parsed_uri.to_s
     end
 
     private
 
-    def url_from_relative(url, current_page_url)
-      if url.start_with?('/')
-        "#{without_path_suffix(resolved_base_url)}#{url}"
-      elsif url.start_with?('../')
-        "#{url_from_dotted_url(url, current_page_url)}"
-      else
-        "#{with_path_suffix(resolved_base_url)}#{url}"
-      end
-    end
-
-    def url_from_dotted_url(url, current_page_url)
-      absolute_url = with_path_suffix(current_page_url.dup)
-      found_dots   = without_path_suffix(url).scan('../').length
-      removed_dots = 0
-      max_levels   = 4
-      while found_dots >= removed_dots && max_levels > removed_dots
-        index = absolute_url.rindex('/') or break
-        absolute_url = absolute_url[0..(index - 1)]
-        removed_dots += 1
-      end
-      "#{with_path_suffix(absolute_url)}#{url.gsub('../', '')}"
-    end
-
-    def with_path_suffix(passed_url)
-      url = passed_url.dup
-      url.end_with?('/') ? url : url << '/'
-    end
-
-    def without_path_suffix(passed_url)
-      url = passed_url.dup
-      url.end_with?('/') ? url[0...(url.length - 1)] : url
-    end
-
     def eligible_url?(href)
       return false if href.nil? || href.empty?
       dont_start   = %w(javascript: callto: mailto: tel: skype: facetime: wtai: #)
       dont_include = %w(/email-protection#)
-      dont_end     = %w(.zip .rar .pdf .exe .dmg .pkg .dpkg .bat)
+      err_include  = %w(/sorry/IndexRedirect?)
+      dont_end     = %w(.zip .rar .json .pdf .exe .dmg .pkg .dpkg .bat)
 
+      err_include.each  { |pattern| fail TOO_MANY_REQUEST_MSG if href.include?(pattern) }
       dont_start.each   { |pattern| return false if href.start_with?(pattern) }      
       dont_include.each { |pattern| return false if href.include?(pattern) }
       dont_end.each     { |pattern| return false if href.end_with?(pattern) }
       true
     end
-
-    def same_domain?(first, second)
-      first.include?(second)
-    end
   end
 end

data/lib/site_mapper/crawler.rb

@@ -4,20 +4,39 @@ require 'nokogiri'
 module SiteMapper
   # Crawls a given site.
   class Crawler
+    # Default options
+    OPTIONS = {
+      resolve:      false,
+      sleep_length: 0.5,
+      max_requests: Float::INFINITY
+    }
+
     # @param [String] url base url for crawler
     # @param [Hash] options hash, resolve key (optional false by default)
+    # add user_agent key to specify custom User-agent
+    # @example Create crawler with custom User-agent
+    #    Crawler.new('example.com', user_agent: 'MyUserAgent')
+    # @example Create crawler and resolve all urls
+    #    Crawler.new('example.com', resolve: true)
+    # @example Create crawler and sleep 1 second between each request
+    #    Crawler.new('example.com', sleep_length: 1)
+    # @example Create crawler and perform max 3 requests
+    #    Crawler.new('example.com', max_requests: 3)
     def initialize(url, options = {})
       @base_url    = Request.resolve_url(url)
-      @options     = { resolve: false }.merge(options)
+      @options     = OPTIONS.dup.merge(options)
+      @user_agent  = @options.fetch(:user_agent)
       @crawl_url   = CrawlUrl.new(@base_url)
       @fetch_queue = CrawlQueue.new
       @processed   = Set.new
       @robots      = nil
     end
 
+    # See documentation for the instance variant of this method.
+    # @return [Array] with links.
     # @see #collect_urls
-    def self.collect_urls(base_url)
-      new(base_url).collect_urls { |url| yield(url) }
+    def self.collect_urls(*args)
+      new(*args).collect_urls { |url| yield(url) }
     end
 
     # Collects all links on domain for domain.
@@ -32,13 +51,16 @@ module SiteMapper
     #   end
     def collect_urls
       @fetch_queue << @crawl_url.resolved_base_url
-      until @fetch_queue.empty?
+      until @fetch_queue.empty? || @processed.length >= @options[:max_requests]
         url = @fetch_queue.pop
         yield(url)
-        page_links(url)
+        page_urls_for(url)
       end
-      Logger.log "Crawling finished, #{@processed.length} links found"
-      @processed.to_a
+      result = @processed + @fetch_queue
+      Logger.log "Crawling finished:"
+      Logger.log "Processed links: #{@processed.length}"
+      Logger.log "Found links:     #{result.length}"
+      result.to_a
     rescue Interrupt, IRB::Abort
       Logger.err_log 'Crawl interrupted.'
       @fetch_queue.to_a
@@ -46,12 +68,13 @@ module SiteMapper
 
     private
 
-    def page_links(get_url)
-      Logger.log "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
-      link_elements = Request.get_page(get_url).css('a') rescue []
-      @processed << get_url
+    def page_urls_for(current_url)
+      Logger.log "Queue length: #{@fetch_queue.length}, Parsing: #{current_url}"
+      link_elements = Request.document(current_url, user_agent: @options[:user_agent]).css('a')
+      wait
+      @processed << current_url
       link_elements.each do |page_link|
-        url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
+        url = @crawl_url.absolute_url_from(page_link.attr('href'), current_url)
         @fetch_queue << url if url && eligible_for_queue?(resolve(url))
       end
     end
@@ -62,7 +85,7 @@ module SiteMapper
 
     def robots
       return @robots unless @robots.nil?
-      robots_body  = Request.get_response_body("#{@base_url}/robots.txt")
+      robots_body  = Request.response_body("#{@base_url}/robots.txt", user_agent: @options[:user_agent])
       @robots      = Robots.new(robots_body, URI.parse(@base_url).host, SiteMapper::USER_AGENT)
       @robots
     end
@@ -71,13 +94,17 @@ module SiteMapper
       @options[:resolve] ? Request.resolve_url(url) : url
     end
 
+    def wait
+      sleep @options[:sleep_length]
+    end
+
     # Queue of urls to be crawled.
     class CrawlQueue
       # @return [Set] that exends EnumerablePop module
       def self.new
         Set.new.extend(EnumerablePop)
       end
-      
+
       # Add pop method when added to class.
       # The class that extends this module need to implement #first and #delete.
       module EnumerablePop
@@ -90,5 +117,5 @@ module SiteMapper
         end
       end
     end
-  end  
+  end
 end

data/lib/site_mapper/logger.rb

@@ -1,6 +1,9 @@
 module SiteMapper
   # Handles logging
   class Logger
+
+    # Choose what logger to use by type.
+    # @return [Object] returns the appropiate logger.
     # @param [Symbol] type of logger class to be used
     def self.use_logger_type(type)
       fail 'Logger already set' if defined?(@@log)
@@ -15,18 +18,22 @@ module SiteMapper
       @@log
     end
 
+    # Choose what logger to use.
+    # @return [Object] returns logger.
     # @param [Class, #log, #err_log] logger a logger class
     def self.use_logger(logger)
       fail 'Logger already set' if defined?(@@log)
       @@log = logger
     end
 
+    # Send a message to the logger
     # @param [String] msg to be logged
     def self.log(msg)
       @@log ||= use_logger_type(:default)
       @@log.log(msg)
     end
 
+    # Send an error message to the logger
     # @param [String] err_msg to be logged
     def self.err_log(err_msg)
       @@log ||= use_logger_type(:default)
@@ -35,11 +42,13 @@ module SiteMapper
 
     # Log to terminal.
     module SystemOutLogger
+      # Log to STDOUT
       # @param [String] msg to be logged to STDOUT
       def self.log(msg)
         STDOUT.puts(msg)
       end
 
+      # Log to STDERR
       # @param [String] msg to be logged to STDERR
       def self.err_log(msg)
         STDERR.puts("[ERROR] #{msg}")
@@ -48,8 +57,10 @@ module SiteMapper
 
     # Don't log
     module NilLogger
+      # Don't log
       # @param [String] msg to be ignored
       def self.log(msg);end
+      # Don't error log
       # @param [String] msg to be ignored
       def self.err_log(msg);end
     end

data/lib/site_mapper/request.rb

@@ -11,61 +11,62 @@ module SiteMapper
     class << self
       # Given an URL get it then parse it with Nokogiri::HTML.
       # @param [String] url
+      # @param [Hash] options
       # @return [Nokogiri::HTML] a nokogiri HTML object
-      def get_page(url)
-        Nokogiri::HTML(Request.get_response_body(url))
+      def document(url, options = {})
+        Nokogiri::HTML(Request.response_body(url, options))
       end
 
       # Given an URL get the response.
       # @param [String] url
-      # @param [Boolean] resolve (optional and false by default)
+      # @param [Hash] options
       # @return [Net::HTTPOK] if response is successfull, raises error otherwise
       # @example get example.com and resolve the URL
-      #    Request.get_response('example.com', true)
+      #    Request.response('example.com', resolve: true)
       # @example get example.com and do *not* resolve the URL
-      #    Request.get_response('http://example.com')
-      #    Request.get_response('http://example.com', false)
-      def get_response(url, resolve = false)
-        resolved_url = resolve ? resolve_url(url) : url
+      #    Request.response('http://example.com')
+      # @example get example.com and resolve the URL
+      #    Request.response('http://example.com', resolve: true)
+      # @example get example.com and resolve the URL and use a custom User-Agent
+      #    Request.response('http://example.com', resolve: true, user_agent: 'MyUserAgent')
+      def response(url, options = {})
+        options = {
+          resolve: false,
+          user_agent: SiteMapper::USER_AGENT
+        }.merge(options)
+        resolved_url = options[:resolve] ? resolve_url(url) : url
         uri          = URI.parse(resolved_url)
         http         = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true if resolved_url.include?('https://')
+        http.use_ssl = true if resolved_url.start_with?('https://')
 
         request = Net::HTTP::Get.new(uri.request_uri)
-        request['User-Agent'] = SiteMapper::USER_AGENT
+        request['User-Agent'] = options[:user_agent]
         http.request(request)
       end
 
       # Get response body, rescues with nil if an exception is raised.
-      # @see Request#get_response
-      def get_response_body(*args)
-        get_response(*args).body rescue nil
+      # @see Request#response
+      def response_body(*args)
+        response(*args).body
       end
 
       # Resolve an URL string and follows redirects.
       # if the URL can't be resolved the original URL is returned.
-      # @param [String] url
-      # @param [Hash] options hash, with_query key (optional and true by default)
+      # @param [String] url to resolve
       # @return [String] a URL string that potentially is a redirected URL
       # @example Resolve google.com
       #    resolve_url('google.com')
       #    # => 'https://www.google.com'
-      def resolve_url(url, options = {})
-        options  = { with_query: true }.merge(options)
+      def resolve_url(url)
         resolved = UrlResolver.resolve(url)
-        resolved = remove_query(resolved) unless options[:with_query]
+        resolved = resolved.prepend('http://') unless has_protocol?(resolved)
         resolved
       end
 
-      # Removes query string from URL string.
-      # @param [String] url
-      # @return [String] an URL string without query
-      # @example Removes query string
-      #    remove_query('example.com/path?q=keyword')
-      #    # => 'example.com/path'
-      def remove_query(url)
-        index = url.index('?')
-        index.nil? ? url : url[0...index]
+      private
+
+      def has_protocol?(url)
+        url.start_with?('https://') || url.start_with?('http://')
       end
     end
   end

data/lib/site_mapper/robots.rb

@@ -6,6 +6,7 @@ module SiteMapper
   class Robots
     # Parses robots.txt
     class ParsedRobots
+      # Initializes ParsedRobots
       def initialize(body, user_agent)
         @other     = {}
         @disallows = {}
@@ -61,7 +62,7 @@ module SiteMapper
         path    = uri.request_uri
 
         user_agent.downcase!
-        
+
         @disallows.each do |key, value|
           if user_agent =~ key
             value.each do |rule|
@@ -71,9 +72,9 @@ module SiteMapper
             end
           end
         end
-        
+
         @allows.each do |key, value|
-          unless allowed      
+          unless allowed
             if user_agent =~ key
               value.each do |rule|
                 if path =~ rule
@@ -93,7 +94,8 @@ module SiteMapper
         agent = to_regex(agent.downcase) if user_agent.is_a?(String)
         @delays[agent]
       end
-      
+
+      # Return key/value paris with unknown meaning.
       # @return [Hash] key/value pairs from robots.txt
       def other_values
         @other
@@ -103,9 +105,11 @@ module SiteMapper
       def sitemaps
         @sitemaps
       end
-        
+
       protected
-      
+
+      # @return [Regex] regex from pattern
+      # @param [String] pattern to compile to Regex
       def to_regex(pattern)
         return /should-not-match-anything-123456789/ if pattern.strip.empty?
         pattern = Regexp.escape(pattern)
@@ -123,7 +127,7 @@ module SiteMapper
       @user_agent = user_agent
       @parsed     = {}
     end
-    
+
     # @param [String, URI] uri String or URI to check
     # @return [Boolean] true if uri is allowed to be crawled
     # @example Check if http://www.google.com/googlesites is allowed to be crawled
@@ -134,8 +138,6 @@ module SiteMapper
       host = uri.host
       @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
       @parsed[host].allowed?(uri, @user_agent)
-    rescue
-      true
     end
 
     # @return [Array] array of sitemaps defined in robots.txt
@@ -146,10 +148,8 @@ module SiteMapper
       host = @hostname
       @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
       @parsed[host].sitemaps
-    rescue
-      []
     end
-    
+
     # @param [String, URI] uri String or URI get other_values from
     # @return [Hash] key/value pairs from robots.txt
     # @example Get other values for google.com
@@ -159,12 +159,10 @@ module SiteMapper
       host = @hostname
       @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
       @parsed[host].other_values
-    rescue
-      {}
     end
 
-    private 
-    
+    private
+
     def to_uri(uri)
       uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
       uri

data/lib/site_mapper/version.rb

@@ -1,4 +1,4 @@
 module SiteMapper
   # Gem version
-  VERSION = '0.0.10'
+  VERSION = '0.0.12'
 end

metadata

@@ -1,125 +1,125 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.12
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-24 00:00:00.000000000 Z
+date: 2015-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: url_resolver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
 description: Map all links on a given site.
@@ -131,13 +131,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/site_mapper
+- lib/site_mapper.rb
 - lib/site_mapper/crawl_url.rb
 - lib/site_mapper/crawler.rb
 - lib/site_mapper/logger.rb
 - lib/site_mapper/request.rb
 - lib/site_mapper/robots.rb
 - lib/site_mapper/version.rb
-- lib/site_mapper.rb
 homepage: https://github.com/buren/site_mapper
 licenses:
 - MIT
@@ -148,17 +148,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Map all links on a given site.