site_mapper 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6870976b0c732915cd3077c0be8bbfd6f92d7f45
-  data.tar.gz: 1f2949d91774c87dd2f156b562ffb9e532f924fe
+  metadata.gz: 472f6b099e7d4c2fe67862faf59b703c65e39a8c
+  data.tar.gz: ce4b315b256fdded26a12665f50d60dccf127ea3
 SHA512:
-  metadata.gz: d736d6c6d62c55e9f38417bb9061dfb19de907ef00d2e41cb31bd1d190d92fd16b319d02483673c333cfcc6cf6feed720906b2ba914957f13f602fc595ef2223
-  data.tar.gz: 69661165ac90ede2611f2d24ba43c58c1eed7b17c5405b23d00df0de023e4896210f75ab152e8d2311832ee4aa828d17c59887e0e943e00b8a0daa34d2d3b1ad
+  metadata.gz: 971616d2f3dd773e63be7d01259582099fd285538b52e04cbcc8055958d98fa57644d84e5546408401c390c6fc56ff70b9ecfc13b2c97bd15660a0a81dd98107
+  data.tar.gz: e27f9ae323e7a536696625071b30b9d046bbaced373e9ae798cd2083556c27bf228ff197a154a2291900b89fc4510264a4565ad1592a86a330f83989f2914a8f

data/lib/site_mapper/crawl_url.rb

@@ -1,5 +1,5 @@
 module SiteMapper
-  # Crawl URL formatter
+  # Crawl URL formatter.
   class CrawlUrl
     attr_reader :resolved_base_url, :base_hostname
 
@@ -11,7 +11,7 @@ module SiteMapper
 
     # Given a link it constructs the absolute path,
     # if valid URL & URL has same domain as @resolved_base_url.
-    # @return [String] with absolute path to resource.
+    # @return [String] with absolute path to resource
     # @param [String, String] raw_url from link element and current page URL
     # @example Construct absolute URL for '/path', example.com
     #   cu = CrawlUrl.new('example.com')

data/lib/site_mapper/crawler.rb

@@ -8,12 +8,13 @@ module SiteMapper
       'User-Agent' => "SiteMapper/#{SiteMapper::VERSION} (+#{CRAWLER_INFO_LINK})"
     }
 
-    def initialize(url, resolve = false)
+    def initialize(url, resolve: false)
       base_url     = Request.resolve_url(url)
       @options     = { resolve: resolve }
       @crawl_url   = CrawlUrl.new(base_url)
       @fetch_queue = CrawlQueue.new
       @processed   = Set.new
+      @robots      = Robots.new(base_url, HEADERS_HASH['User-Agent'])
     end
 
     # @see #collect_urls
@@ -21,7 +22,7 @@ module SiteMapper
       new(base_url).collect_urls { |url| yield(url) }
     end
 
-    # Collects all links on domain for domain
+    # Collects all links on domain for domain.
     # @return [Array] with links.
     # @example URLs for example.com
     #   crawler = Crawler.new('example.com')
@@ -52,14 +53,15 @@ module SiteMapper
       link_elements = Request.get_page(get_url).css('a') rescue []
       @processed << get_url
       link_elements.each do |page_link|
-        absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
-        if absolute_url
-          url = resolve(absolute_url)
-          @fetch_queue << url unless @processed.include?(url)
-        end
+        url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
+        @fetch_queue << url if url && eligible_for_queue?(resolve(url))
       end
     end
 
+    def eligible_for_queue?(url)
+      @robots.allowed?(url) && !@processed.include?(url)
+    end
+
     def resolve(url)
       @options[:resolve] ? Request.resolve_url(url) : url
     end

data/lib/site_mapper/request.rb

@@ -1,13 +1,16 @@
 require 'url_resolver' # TODO: Allow users to use any resolver
 
 module SiteMapper
+  # Get webpage wrapper.
   class Request
     INFO_LINK  = 'https://rubygems.org/gems/site_mapper'
     USER_AGENT = "SiteMapper/#{SiteMapper::VERSION} (+#{INFO_LINK})"
 
     class << self
-      def get_page(url, document_type = :html)
-        Nokogiri::HTML(Request.get_response(url).body)
+      # Given an URL get it then parse it with Nokogiri::HTML.
+      # @return [Nokogiri::HTML] a nokogiri HTML object
+      def get_page(url)
+        Nokogiri::HTML(Request.get_response_body(url))
       end
 
       def get_response(url, resolve = false)
@@ -21,12 +24,29 @@ module SiteMapper
         http.request(request)
       end
 
+      # Get response body, rescues with nil if an exception is raised
+      # @see #get_response
+      def get_response_body(*args)
+        get_response(*args).body rescue nil
+      end
+
+      # Resolve an URL string and follows redirects
+      # if the URL can't be resolved the original URL is returned.
+      # @return [String] a URL string that potentially is a redirected URL
+      # @example Resolve google.com
+      #    resolve_url('google.com')
+      #    # => 'https://www.google.com'
       def resolve_url(url, with_query: true)
         resolved = UrlResolver.resolve(url)
         resolved = remove_query(resolved) unless with_query
         resolved
       end
 
+      # Removes query string from URL string.
+      # @return [String] an URL string without query
+      # @example Removes query string
+      #    remove_query('example.com/path?q=keyword')
+      #    # => 'example.com/path'
       def remove_query(url)
         index = url.index('?')
         index.nil? ? url : url[0...index]

data/lib/site_mapper/robots.rb

@@ -0,0 +1,144 @@
+# Based on: https://rubygems.org/gems/robots, v0.10.1
+module SiteMapper
+  # Provided a base URL it checks whether a given URL is
+  # allowed to be crawled according to /robots.txt
+  # @see https://rubygems.org/gems/robots
+  class Robots
+    # Parses robots.txt
+    class ParsedRobots
+      def initialize(body, user_agent)
+        @other     = {}
+        @disallows = {}
+        @allows    = {}
+        @delays    = {}
+        parse(body)
+      end
+
+      # Parse robots.txt body.
+      def parse(body)
+        agent = /.*/
+        body  = body || "User-agent: *\nAllow: /\n"
+        body  = body.downcase
+        body.each_line.each do |line|
+          next if line =~ /^\s*(#.*|$)/
+          arr   = line.split(':')
+          key   = arr.shift
+          value = arr.join(':').strip
+          value.strip!
+          case key
+          when 'user-agent'
+            agent = to_regex(value)
+          when 'allow'
+            @allows[agent] ||= []
+            @allows[agent] << to_regex(value)
+          when 'disallow'
+            @disallows[agent] ||= []
+            @disallows[agent] << to_regex(value)
+          when 'crawl-delay'
+            @delays[agent] = value.to_i
+          else
+            @other[key] ||= []
+            @other[key] << value
+          end
+        end
+        @parsed = true
+      end
+      
+      # @return [Boolean] true if uri is allowed to be crawled
+      # @example Check if http://www.google.com/googlesites is allowed to be crawled
+      #    uri = URI.parse('http://www.google.com/googlesites')
+      #    robots.allowed?(uri, 'SiteMapper') # => false (as of 2014-10-22)
+      def allowed?(uri, user_agent)
+        return true unless @parsed
+        allowed = true
+        path    = uri.request_uri
+        
+        @disallows.each do |key, value|
+          if user_agent =~ key
+            value.each do |rule|
+              if path =~ rule
+                allowed = false
+              end
+            end
+          end
+        end
+        
+        @allows.each do |key, value|
+          unless allowed      
+            if user_agent =~ key
+              value.each do |rule|
+                if path =~ rule
+                  allowed = true
+                end
+              end
+            end
+          end
+        end
+        allowed
+      end
+      
+      # @return [Hash] key/value pairs from robots.txt
+      def other_values
+        @other
+      end
+        
+      protected
+      
+      def to_regex(pattern)
+        return /should-not-match-anything-123456789/ if pattern.strip.empty?
+        pattern = Regexp.escape(pattern)
+        pattern.gsub!(Regexp.escape('*'), '.*')
+        Regexp.compile("^#{pattern}")
+      end
+    end
+
+    def initialize(url, user_agent)
+      @user_agent = user_agent
+      @parsed     = {}
+      @robots_txt = Request.get_response_body("#{url}/robots.txt", true)
+    end
+    
+    # @return [Boolean] true if uri is allowed to be crawled
+    # @example Check if http://www.google.com/googlesites is allowed to be crawled
+    #    robots = Robots.new('google.com', 'SiteMapper')
+    #    robots.allowed?('http://www.google.com/googlesites') # => false (as of 2014-10-22)
+    def allowed?(uri)
+      uri  = to_uri(uri)
+      host = uri.host
+      @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
+      @parsed[host].allowed?(uri, @user_agent)
+    rescue
+      true
+    end
+
+    # @return [Array] array of sitemaps defined in robots.txt
+    # @example Get sitemap for google.com
+    #    robots = Robots.new('google.com', 'SiteMapper')
+    #    robots.sitemaps
+    def sitemaps
+      uri    = to_uri(uri)
+      values = other_values(uri.host)
+      values['sitemap'] or []
+    rescue
+      []
+    end
+    
+    # @return [Hash] key/value pairs from robots.txt
+    # @example Get other values for google.com
+    #    robots = Robots.new('google.com', 'SiteMapper')
+    #    robots.other_values
+    def other_values(uri)
+      uri  = to_uri(uri)
+      host = uri.host
+      @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
+      @parsed[host].other_values
+    end
+
+    private 
+    
+    def to_uri(uri)
+      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+      uri
+    end
+  end
+end

data/lib/site_mapper/version.rb

@@ -1,3 +1,3 @@
 module SiteMapper
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
 end

data/lib/site_mapper.rb

@@ -3,6 +3,7 @@ require 'net/http'
 
 require 'site_mapper/version'
 require 'site_mapper/request'
+require 'site_mapper/robots'
 require 'site_mapper/crawler'
 require 'site_mapper/crawl_url'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Jacob Burenstam
@@ -108,7 +108,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.7'
-description: Find all links on domain to domain.
+description: Map all links on a given site.
 email:
 - burenstam@gmail.com
 executables:
@@ -117,11 +117,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/site_mapper
-- lib/site_mapper.rb
 - lib/site_mapper/crawl_url.rb
 - lib/site_mapper/crawler.rb
 - lib/site_mapper/request.rb
+- lib/site_mapper/robots.rb
 - lib/site_mapper/version.rb
+- lib/site_mapper.rb
 homepage: https://github.com/buren/site_mapper
 licenses:
 - MIT
@@ -142,9 +143,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
-summary: Find all links on domain to domain
+summary: Map all links on a given site.
 test_files: []
 has_rdoc: