spidr_epg 1.0.0

data/lib/spidr/auth_credential.rb

@@ -0,0 +1,28 @@
+module Spidr
+  #
+  # Represents HTTP Authentication credentials for a website.
+  #
+  class AuthCredential
+
+    # The username
+    attr_reader :username
+
+    # The password
+    attr_reader :password
+
+    #
+    # Creates a new credential used for authentication.
+    #
+    # @param [String] username
+    #   The username for the credential.
+    #
+    # @param [String] password
+    #   The password for the credential.
+    #
+    def initialize(username,password)
+      @username = username
+      @password = password
+    end
+
+  end
+end

data/lib/spidr/auth_store.rb

@@ -0,0 +1,161 @@
+require 'spidrs/extensions/uri'
+require 'spidrs/auth_credential'
+require 'spidrs/page'
+
+require 'base64'
+
+module Spidr
+  #
+  # Stores {AuthCredential} objects organized by a website's scheme,
+  # host-name and sub-directory.
+  #
+  class AuthStore
+
+    #
+    # Creates a new auth store.
+    #
+    # @since 0.2.2
+    #
+    def initialize
+      @credentials = {}
+    end
+
+    # 
+    # Given a URL, return the most specific matching auth credential.
+    #
+    # @param [URI] url
+    #   A fully qualified url including optional path.
+    #
+    # @return [AuthCredential, nil]
+    #   Closest matching {AuthCredential} values for the URL,
+    #   or `nil` if nothing matches.
+    #
+    # @since 0.2.2
+    #
+    def [](url)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      key = [url.scheme, url.host, url.port]
+      paths = @credentials[key]
+
+      return nil unless paths
+
+      # longest path first
+      ordered_paths = paths.keys.sort_by { |key| key.length }.reverse
+
+      # directories of the path
+      path_dirs = URI.expand_path(url.path).split('/')
+
+      ordered_paths.each do |path|
+        return paths[path] if path_dirs[0,path.length] == path
+      end
+
+      return nil
+    end
+
+    # 
+    # Add an auth credential to the store for supplied base URL.
+    #
+    # @param [URI] url_base
+    #   A URL pattern to associate with a set of auth credentials.
+    #
+    # @param [AuthCredential]
+    #   The auth credential for this URL pattern.
+    #
+    # @return [AuthCredential]
+    #   The newly added auth credential.
+    #
+    # @since 0.2.2
+    #
+    def []=(url,auth)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      # normalize the URL path
+      path = URI.expand_path(url.path)
+
+      key = [url.scheme, url.host, url.port]
+
+      @credentials[key] ||= {}
+      @credentials[key][path.split('/')] = auth
+      return auth
+    end
+
+    #
+    # Convenience method to add username and password credentials
+    # for a named URL.
+    #
+    # @param [URI] url
+    #   The base URL that requires authorization.
+    #
+    # @param [String] username
+    #   The username required to access the URL.
+    #
+    # @param [String] password
+    #   The password required to access the URL.
+    #
+    # @return [AuthCredential]
+    #   The newly added auth credential.
+    #
+    # @since 0.2.2
+    #
+    def add(url,username,password)
+      self[url] = AuthCredential.new(username,password)
+    end
+
+    #
+    # Returns the base64 encoded authorization string for the URL
+    # or `nil` if no authorization exists.
+    #
+    # @param [URI] url
+    #   The url.
+    #
+    # @return [String, nil]
+    #   The base64 encoded authorizatio string or `nil`.
+    #
+    # @since 0.2.2
+    #
+    def for_url(url)
+      if (auth = self[url])
+        return Base64.encode64("#{auth.username}:#{auth.password}")
+      end
+    end
+
+    # 
+    # Clear the contents of the auth store.
+    #
+    # @return [AuthStore]
+    #   The cleared auth store.
+    #
+    # @since 0.2.2
+    #
+    def clear!
+      @credentials.clear
+      return self
+    end
+
+    #
+    # Size of the current auth store (number of URL paths stored).
+    #
+    # @return [Integer]
+    #   The size of the auth store.
+    #
+    # @since 0.2.2
+    #
+    def size
+      @credentials.inject(0) { |res, arr| res + arr[1].length }
+    end
+
+    #
+    # Inspects the auth store.
+    #
+    # @return [String]
+    #   The inspected version of the auth store.
+    #
+    def inspect
+      "#<#{self.class}: #{@credentials.inspect}>"
+    end
+
+  end
+end

data/lib/spidr/body.rb

@@ -0,0 +1,98 @@
+require 'nokogiri'
+
+module Spidr
+  module Body
+    #
+    # The body of the response.
+    #
+    # @return [String]
+    #   The body of the response.
+    #
+    def body
+      (response.body || '')
+    end
+
+    #
+    # Returns a parsed document object for HTML, XML, RSS and Atom pages.
+    #
+    # @return [Nokogiri::HTML::Document, Nokogiri::XML::Document, nil]
+    #   The document that represents HTML or XML pages.
+    #   Returns `nil` if the page is neither HTML, XML, RSS, Atom or if
+    #   the page could not be parsed properly.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
+    #
+    def doc
+      unless body.empty?
+        begin
+          if html?
+            @doc ||= Nokogiri::HTML(body, @url.to_s, content_charset)
+          elsif (rss? || atom? || xml? || xsl?)
+            @doc ||= Nokogiri::XML(body, @url.to_s, content_charset)
+          end
+        rescue
+        end
+      end
+    end
+
+    #
+    # Searches the document for XPath or CSS Path paths.
+    #
+    # @param [Array<String>] paths
+    #   CSS or XPath expressions to search the document with.
+    #
+    # @return [Array]
+    #   The matched nodes from the document.
+    #   Returns an empty Array if no nodes were matched, or if the page
+    #   is not an HTML or XML document.
+    #
+    # @example
+    #   page.search('//a[@href]')
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Node.html#M000239
+    #
+    def search(*paths)
+      if doc
+        doc.search(*paths)
+      else
+        []
+      end
+    end
+
+    #
+    # Searches for the first occurrence an XPath or CSS Path expression.
+    #
+    # @return [Nokogiri::HTML::Node, Nokogiri::XML::Node, nil]
+    #   The first matched node. Returns `nil` if no nodes could be matched,
+    #   or if the page is not a HTML or XML document.
+    #
+    # @example
+    #   page.at('//title')
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Node.html#M000251
+    #
+    def at(*arguments)
+      if doc
+        doc.at(*arguments)
+      end
+    end
+
+    alias / search
+    alias % at
+
+    #
+    # The title of the HTML page.
+    #
+    # @return [String]
+    #   The inner-text of the title element of the page.
+    #
+    def title
+      if (node = at('//title'))
+        node.inner_text
+      end
+    end
+
+    alias to_s body
+  end
+end

data/lib/spidr/cookie_jar.rb

@@ -0,0 +1,202 @@
+require 'spidrs/page'
+
+require 'set'
+
+module Spidr
+  #
+  # Stores HTTP Cookies organized by host-name.
+  #
+  class CookieJar
+
+    include Enumerable
+
+    #
+    # Creates a new Cookie Jar object.
+    #
+    # @since 0.2.2
+    #
+    def initialize
+      @params = {}
+
+      @dirty   = Set[]
+      @cookies = {}
+    end
+
+    #
+    # Enumerates over the host-name and cookie value pairs in the
+    # cookie jar.
+    #
+    # @yield [host, cookie]
+    #   If a block is given, it will be passed each host-name and cookie
+    #   value pair.
+    #
+    # @yieldparam [String] host
+    #   The host-name that the cookie is bound to.
+    #
+    # @yieldparam [String] cookie
+    #   The cookie value.
+    #
+    # @since 0.2.2
+    #
+    def each(&block)
+      @params.each(&block)
+    end
+
+    # 
+    # Return all relevant cookies in a single string for the 
+    # named host or domain (in browser request format).
+    #
+    # @param [String] host
+    #   Host or domain name for cookies.
+    #
+    # @return [String, nil]
+    #   The cookie values or `nil` if the host does not have a cookie in the
+    #   jar.
+    #
+    # @since 0.2.2
+    #
+    def [](host)
+      @params[host] ||= {}
+    end
+
+    # 
+    # Add a cookie to the jar for a particular domain.
+    #
+    # @param [String] host
+    #   Host or domain name to associate with the cookie.
+    #
+    # @param [Hash{String => String}] cookies
+    #   Cookie params.
+    #
+    # @since 0.2.2
+    #
+    def []=(host,cookies)
+      collected = self[host]
+
+      cookies.each do |key,value|
+        if collected[key] != value
+          collected.merge!(cookies)
+          @dirty << host
+
+          break
+        end
+      end
+
+      return cookies
+    end
+
+    #
+    # Retrieve cookies for a domain from a page response header.
+    #
+    # @param [Page] page
+    #   The response page from which to extract cookie data.
+    #
+    # @return [Boolean]
+    #   Specifies whether cookies were added from the page.
+    #
+    # @since 0.2.2
+    #
+    def from_page(page)
+      cookies = page.cookie_params
+
+      unless cookies.empty?
+        self[page.url.host] = cookies
+        return true
+      end
+
+      return false
+    end
+
+    #
+    # Returns the pre-encoded Cookie for a given host.
+    #
+    # @param [String] host
+    #   The name of the host.
+    #
+    # @return [String]
+    #   The encoded Cookie.
+    #
+    # @since 0.2.2
+    #
+    def for_host(host)
+      if @dirty.include?(host)
+        values = []
+
+        cookies_for_host(host).each do |name,value|
+          values << "#{name}=#{value}"
+        end
+
+        @cookies[host] = values.join('; ')
+        @dirty.delete(host)
+      end
+
+      return @cookies[host]
+    end
+
+    #
+    # Returns raw cookie value pairs for a given host. Includes cookies set on
+    # parent domain(s).
+    #
+    # @param [String] host
+    #   The name of the host.
+    #
+    # @return [Hash{String => String}]
+    #   Cookie params.
+    #
+    # @since 0.2.7
+    #
+    def cookies_for_host(host)
+      host_cookies = (@params[host] || {})
+      sub_domains  = host.split('.')
+
+      while sub_domains.length > 2
+        sub_domains.shift
+
+        if (parent_cookies = @params[sub_domains.join('.')])
+          parent_cookies.each do |name,value|
+            # copy in the parent cookies, only if they haven't been
+            # overridden yet.
+            unless host_cookies.has_key?(name)
+              host_cookies[name] = value
+            end
+          end
+        end
+      end
+
+      return host_cookies
+    end
+
+    # 
+    # Clear out the jar, removing all stored cookies.
+    #
+    # @since 0.2.2
+    #
+    def clear!
+      @params.clear
+
+      @dirty.clear
+      @cookies.clear
+      return self
+    end
+
+    #
+    # Size of the current cookie jar store.
+    #
+    # @since 0.2.2
+    #
+    def size
+      @params.size
+    end
+
+    #
+    # Inspects the cookie jar.
+    #
+    # @return [String]
+    #   The inspected version of the cookie jar.
+    #
+    def inspect
+      "#<#{self.class}: #{@params.inspect}>"
+    end
+
+  end
+end