site_mapper 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b52f224f6a1dbfdc6207741e9efe92fd6e4adbc5
-  data.tar.gz: dc70e4ba139a385164aa122e17e61e06bf469614
+  metadata.gz: d2f7c3a2410e0cb508297f6fb229c1e5ec5e8f98
+  data.tar.gz: 6495918729c0e0d4c8788c4eb126c829b72dc17b
 SHA512:
-  metadata.gz: 6546905269282bb362050fb41c679a42aa7ea91f849425c31b0493f1f6c1727e043e7e147ab51042cafe18afd9e092b91293247c5f3de6f57d8fb13d0a58ca06
-  data.tar.gz: f1821f346544348ac708ed2582538d7c56a7151058a0880a8cb6cf14a93b23481c458e9a512a630e7bf25b0bd768739511245f3eb40b8d8a90504e8cb4f378d1
+  metadata.gz: 80fceb2fd976b3c0e6ffe2f7237ffe9645257a271b99ca7f919e147a132fd09275862ba8c0563924e533e6b5049cd192726a06c48a8c4a4efa1b61b0af2c1e55
+  data.tar.gz: 9ff4ef13486b564bc9f8b3d60c42c248c26ed4245584af3e05a0a907b41ac7bf6bfc44ab5ece3cee91216c240201990c65e0adfa20960b28a0b37f8f6eb6ad4e

data/lib/site_mapper.rb

@@ -9,12 +9,17 @@ require 'site_mapper/crawl_url'
 
 # Find all links on domain to domain
 module SiteMapper
-  # Returns all links found on domain to domain.
+  # SiteMapper info link
+  INFO_LINK  = 'https://rubygems.org/gems/site_mapper'
+  # SiteMapper User-Agent
+  USER_AGENT = "SiteMapper/#{SiteMapper::VERSION} (+#{INFO_LINK})"
+
+  # Map all links on a given site.
   # @return [Array] with links.
   # @param [String] link to domain
   # @example Collect all URLs from example.com
   #   SiteMapper.map('example.com')
-  def self.map(source)
-    Crawler.collect_urls(source) { |url| yield(url) if block_given? }
+  def self.map(link)
+    Crawler.collect_urls(link) { |url| yield(url) if block_given? }
   end
 end

data/lib/site_mapper/crawl_url.rb

@@ -3,6 +3,7 @@ module SiteMapper
   class CrawlUrl
     attr_reader :resolved_base_url, :base_hostname
 
+    # @param [String] base_url
     def initialize(base_url)
       @resolved_base_url = Request.resolve_url(base_url, with_query: false)
       @base_hostname     = URI.parse(@resolved_base_url).hostname
@@ -11,8 +12,9 @@ module SiteMapper
 
     # Given a link it constructs the absolute path,
     # if valid URL & URL has same domain as @resolved_base_url.
+    # @param [String] raw_url url found on page
+    # @param [String] get_url current page url
     # @return [String] with absolute path to resource
-    # @param [String, String] raw_url from link element and current page URL
     # @example Construct absolute URL for '/path', example.com
     #   cu = CrawlUrl.new('example.com')
     #   cu.absolute_url_from('/path', 'example.com/some/path')

data/lib/site_mapper/crawler.rb

@@ -2,19 +2,17 @@ require 'set'
 require 'nokogiri'
 
 module SiteMapper
+  # Crawls a given site.
   class Crawler
-    CRAWLER_INFO_LINK = 'https://rubygems.org/gems/wayback_archiver'
-    HEADERS_HASH      = {
-      'User-Agent' => "SiteMapper/#{SiteMapper::VERSION} (+#{CRAWLER_INFO_LINK})"
-    }
-
+    # @param [String] url base url for crawler
+    # @param [Hash] resolve (optional false by default)
     def initialize(url, resolve: false)
-      base_url     = Request.resolve_url(url)
+      @base_url     = Request.resolve_url(url)
       @options     = { resolve: resolve }
-      @crawl_url   = CrawlUrl.new(base_url)
+      @crawl_url   = CrawlUrl.new(@base_url)
       @fetch_queue = CrawlQueue.new
       @processed   = Set.new
-      @robots      = Robots.new(base_url, HEADERS_HASH['User-Agent'])
+      @robots      = nil
     end
 
     # @see #collect_urls
@@ -59,7 +57,14 @@ module SiteMapper
     end
 
     def eligible_for_queue?(url)
-      @robots.allowed?(url) && !@processed.include?(url)
+      robots.allowed?(url) && !@processed.include?(url)
+    end
+
+    def robots
+      return @robots unless @robots.nil?
+      robots_body  = Request.get_response_body("#{@base_url}/robots.txt")
+      @robots      = Robots.new(robots_body, URI.parse(@base_url).host, SiteMapper::USER_AGENT)
+      @robots
     end
 
     def resolve(url)
@@ -67,12 +72,18 @@ module SiteMapper
     end
   end
 
+  # Queue of urls to be crawled.
   class CrawlQueue
+    # @return [Set] that exends EnumerablePop module
     def self.new
       Set.new.extend(EnumerablePop)
     end
     
+    # Add pop method when added to class.
+    # The class that extends this module need to implement #first and #delete.
     module EnumerablePop
+      # Pop first element from list.
+      # @return [Object] the first object in the list or nil
       def pop
         first_element = first
         delete(first_element)

data/lib/site_mapper/request.rb

@@ -3,16 +3,28 @@ require 'url_resolver' # TODO: Allow users to use any resolver
 module SiteMapper
   # Get webpage wrapper.
   class Request
+    # Request info link
     INFO_LINK  = 'https://rubygems.org/gems/site_mapper'
+    # Request User-Agent
     USER_AGENT = "SiteMapper/#{SiteMapper::VERSION} (+#{INFO_LINK})"
 
     class << self
       # Given an URL get it then parse it with Nokogiri::HTML.
+      # @param [String] url
       # @return [Nokogiri::HTML] a nokogiri HTML object
       def get_page(url)
         Nokogiri::HTML(Request.get_response_body(url))
       end
 
+      # Given an URL get the response.
+      # @param [String] url
+      # @param [Boolean] resolve (optional and false by default)
+      # @return [Net::HTTPOK] if response is successfull, raises error otherwise
+      # @example get example.com and resolve the URL
+      #    Request.get_response('example.com', true)
+      # @example get example.com and do *not* resolve the URL
+      #    Request.get_response('http://example.com')
+      #    Request.get_response('http://example.com', false)
       def get_response(url, resolve = false)
         resolved_url = resolve ? resolve_url(url) : url
         uri          = URI.parse(resolved_url)
@@ -20,18 +32,20 @@ module SiteMapper
         http.use_ssl = true if resolved_url.include?('https://')
 
         request = Net::HTTP::Get.new(uri.request_uri)
-        request['User-Agent'] = USER_AGENT
+        request['User-Agent'] = SiteMapper::USER_AGENT
         http.request(request)
       end
 
-      # Get response body, rescues with nil if an exception is raised
+      # Get response body, rescues with nil if an exception is raised.
       # @see #get_response
       def get_response_body(*args)
         get_response(*args).body rescue nil
       end
 
-      # Resolve an URL string and follows redirects
+      # Resolve an URL string and follows redirects.
       # if the URL can't be resolved the original URL is returned.
+      # @param [String] url
+      # @param [Boolean] with_query (optional and true by default)
       # @return [String] a URL string that potentially is a redirected URL
       # @example Resolve google.com
       #    resolve_url('google.com')
@@ -43,6 +57,7 @@ module SiteMapper
       end
 
       # Removes query string from URL string.
+      # @param [String] url
       # @return [String] an URL string without query
       # @example Removes query string
       #    remove_query('example.com/path?q=keyword')

data/lib/site_mapper/robots.rb

@@ -1,7 +1,7 @@
 # Based on: https://rubygems.org/gems/robots, v0.10.1
 module SiteMapper
   # Provided a base URL it checks whether a given URL is
-  # allowed to be crawled according to /robots.txt
+  # allowed to be crawled according to /robots.txt.
   # @see https://rubygems.org/gems/robots
   class Robots
     # Parses robots.txt
@@ -11,10 +11,12 @@ module SiteMapper
         @disallows = {}
         @allows    = {}
         @delays    = {}
+        @sitemaps  = []
         parse(body)
       end
 
       # Parse robots.txt body.
+      # @param [String] body the webpage body HTML
       def parse(body)
         agent = /.*/
         body  = body || "User-agent: *\nAllow: /\n"
@@ -36,6 +38,8 @@ module SiteMapper
             @disallows[agent] << to_regex(value)
           when 'crawl-delay'
             @delays[agent] = value.to_i
+          when 'sitemap'
+            @sitemaps << value
           else
             @other[key] ||= []
             @other[key] << value
@@ -43,15 +47,20 @@ module SiteMapper
         end
         @parsed = true
       end
-      
+
+      # @param [URI] uri to be checked
+      # @param [String] user_agent to be checked
       # @return [Boolean] true if uri is allowed to be crawled
       # @example Check if http://www.google.com/googlesites is allowed to be crawled
       #    uri = URI.parse('http://www.google.com/googlesites')
-      #    robots.allowed?(uri, 'SiteMapper') # => false (as of 2014-10-22)
+      #    robots.allowed?(uri, 'SiteMapper')
+      #    # => false (as of 2014-10-22)
       def allowed?(uri, user_agent)
         return true unless @parsed
         allowed = true
         path    = uri.request_uri
+
+        user_agent.downcase!
         
         @disallows.each do |key, value|
           if user_agent =~ key
@@ -76,11 +85,24 @@ module SiteMapper
         end
         allowed
       end
+
+      # @param [String] user_agent
+      # @return [Integer] crawl delay for user_agent
+      def crawl_delay(user_agent)
+        agent = user_agent.dup
+        agent = to_regex(agent.downcase) if user_agent.is_a?(String)
+        @delays[agent]
+      end
       
       # @return [Hash] key/value pairs from robots.txt
       def other_values
         @other
       end
+
+      # @return [Array] returns sitemaps defined in robots.txt
+      def sitemaps
+        @sitemaps
+      end
         
       protected
       
@@ -92,12 +114,15 @@ module SiteMapper
       end
     end
 
-    def initialize(url, user_agent)
+    # @param [String] url to fetch /robots.txt from
+    def initialize(robots_txt, hostname, user_agent)
+      @robots_txt = robots_txt
+      @hostname   = hostname
       @user_agent = user_agent
       @parsed     = {}
-      @robots_txt = Request.get_response_body("#{url}/robots.txt", true)
     end
     
+    # @param [String, URI] uri String or URI to check
     # @return [Boolean] true if uri is allowed to be crawled
     # @example Check if http://www.google.com/googlesites is allowed to be crawled
     #    robots = Robots.new('google.com', 'SiteMapper')
@@ -116,22 +141,24 @@ module SiteMapper
     #    robots = Robots.new('google.com', 'SiteMapper')
     #    robots.sitemaps
     def sitemaps
-      uri    = to_uri(uri)
-      values = other_values(uri.host)
-      values['sitemap'] or []
+      host = @hostname
+      @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
+      @parsed[host].sitemaps
     rescue
       []
     end
     
+    # @param [String, URI] uri String or URI get other_values from
     # @return [Hash] key/value pairs from robots.txt
     # @example Get other values for google.com
     #    robots = Robots.new('google.com', 'SiteMapper')
     #    robots.other_values
-    def other_values(uri)
-      uri  = to_uri(uri)
-      host = uri.host
+    def other_values
+      host = @hostname
       @parsed[host] ||= ParsedRobots.new(@robots_txt, @user_agent)
       @parsed[host].other_values
+    rescue
+      {}
     end
 
     private 

data/lib/site_mapper/version.rb

@@ -1,3 +1,4 @@
 module SiteMapper
-  VERSION = '0.0.8'
+  # Gem version
+  VERSION = '0.0.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2014-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri