spidr 0.2.1 → 0.2.2

data/lib/spidr/auth_credential.rb

@@ -0,0 +1,25 @@
+module Spidr
+  class AuthCredential
+
+    # The username
+    attr_reader :username
+
+    # The password
+    attr_reader :password
+
+    #
+    # Creates a new credential used for authentication.
+    #
+    # @param [String] username
+    #   The username for the credential.
+    #
+    # @param [String] password
+    #   The password for the credential.
+    #
+    def initialize(username,password)
+      @username = username
+      @password = password
+    end
+
+  end
+end

data/lib/spidr/auth_store.rb

@@ -0,0 +1,157 @@
+require 'spidr/extensions/uri'
+require 'spidr/auth_credential'
+require 'spidr/page'
+
+require 'base64'
+
+module Spidr
+  class AuthStore
+
+    #
+    # Creates a new auth store.
+    #
+    # @since 0.2.2
+    #
+    def initialize
+      @credentials = {}
+    end
+
+    # 
+    # Given a URL, return the most specific matching auth credential.
+    #
+    # @param [URI] url
+    #   A fully qualified url includig optional path.
+    #
+    # @return [AuthCredential, nil]
+    #   Closest matching {AuthCredential} values for the URL,
+    #   or +nil+ if nothing matches.
+    #
+    # @since 0.2.2
+    #
+    def [](url)
+      # normalize the url
+      url = URI(url) unless url.kind_of?(URI)
+
+      key = [url.scheme, url.host, url.port]
+      paths = @credentials[key]
+
+      return nil unless paths
+
+      # longest path first
+      ordered_paths = paths.keys.sort_by { |key| key.length }.reverse
+
+      # directories of the path
+      path_dirs = URI.expand_path(url.path).split('/')
+
+      ordered_paths.each do |path|
+        return paths[path] if path_dirs[0,path.length] == path
+      end
+
+      return nil
+    end
+
+    # 
+    # Add an auth credential to the store for supplied base URL.
+    #
+    # @param [URI] url_base
+    #   A URL pattern to associate with a set of auth credentials.
+    #
+    # @param [AuthCredential]
+    #   The auth credential for this URL pattern.
+    #
+    # @return [AuthCredential]
+    #   The newly added auth credential.
+    #
+    # @since 0.2.2
+    #
+    def []=(url, auth)
+      # normalize the url
+      url = URI(url) unless url.kind_of?(URI)
+
+      # normalize the URL path
+      path = URI.expand_path(url.path)
+
+      key = [url.scheme, url.host, url.port]
+
+      @credentials[key] ||= {}
+      @credentials[key][path.split('/')] = auth
+      return auth
+    end
+
+    #
+    # Convenience method to add username and password credentials
+    # for a named URL.
+    #
+    # @param [URI] url
+    #   The base URL that requires authorization.
+    #
+    # @param [String] username
+    #   The username required to access the URL.
+    #
+    # @param [String] password
+    #   The password required to access the URL.
+    #
+    # @return [AuthCredential]
+    #   The newly added auth credential.
+    #
+    # @since 0.2.2
+    #
+    def add(url, username, password)
+      self[url] = AuthCredential.new(username, password)
+    end
+
+    #
+    # Returns the base64 encoded authorization string for the URL
+    # or +nil+ if no authorization exists.
+    #
+    # @param [URI] url
+    #   The url.
+    #
+    # @return [String, nil]
+    #   The base64 encoded authorizatio string or +nil+.
+    #
+    # @since 0.2.2
+    #
+    def for_url(url)
+      if (auth = self[url])
+        return Base64.encode64("#{auth.username}:#{auth.password}")
+      end
+    end
+
+    # 
+    # Clear the contents of the auth store.
+    #
+    # @return [AuthStore]
+    #   The cleared auth store.
+    #
+    # @since 0.2.2
+    #
+    def clear!
+      @credentials.clear
+      return self
+    end
+
+    #
+    # Size of the current auth store (number of URL paths stored).
+    #
+    # @return [Integer]
+    #   The size of the auth store.
+    #
+    # @since 0.2.2
+    #
+    def size
+      @credentials.inject(0) { |res, arr| res + arr[1].length }
+    end
+
+    #
+    # Inspects the auth store.
+    #
+    # @return [String]
+    #   The inspected version of the auth store.
+    #
+    def inspect
+      "#<#{self.class}: #{@credentials.inspect}>"
+    end
+
+  end
+end

data/lib/spidr/cookie_jar.rb

@@ -0,0 +1,166 @@
+require 'spidr/page'
+
+require 'set'
+
+module Spidr
+  class CookieJar
+
+    include Enumerable
+
+    #
+    # Creates a new Cookie Jar object.
+    #
+    # @since 0.2.2
+    #
+    def initialize
+      @params = {}
+
+      @dirty = Set[]
+      @cookies = {}
+    end
+
+    #
+    # Enumerates over the host-name and cookie value pairs in the
+    # cookie jar.
+    #
+    # @yield [host, cookie]
+    #   If a block is given, it will be passed each host-name and cookie
+    #   value pair.
+    #
+    # @yieldparam [String] host
+    #   The host-name that the cookie is bound to.
+    #
+    # @yieldparam [String] cookie
+    #   The cookie value.
+    #
+    # @since 0.2.2
+    #
+    def each(&block)
+      @params.each(&block)
+    end
+
+    # 
+    # Return all relevant cookies in a single string for the 
+    # named host or domain (in browser request format).
+    #
+    # @param [String] host
+    #   Host or domain name for cookies.
+    #
+    # @return [String, nil]
+    #   The cookie values or +nil+ if the host does not have a cookie in the
+    #   jar.
+    #
+    # @since 0.2.2
+    #
+    def [](host)
+      @params[host] ||= {}
+    end
+
+    # 
+    # Add a cookie to the jar for a particular domain.
+    #
+    # @param [String] host
+    #   Host or domain name to associate with the cookie.
+    #
+    # @param [Hash{String => String}] cookies
+    #   Cookie params.
+    #
+    # @since 0.2.2
+    #
+    def []=(host,cookies)
+      collected = self[host]
+
+      cookies.each do |key,value|
+        if collected[key] != value
+          collected.merge!(cookies)
+          @dirty << host
+
+          break
+        end
+      end
+
+      return cookies
+    end
+
+    #
+    # Retrieve cookies for a domain from a page response header.
+    #
+    # @param [Page] page
+    #   The response page from which to extract cookie data.
+    #
+    # @return [Boolean]
+    #   Specifies whether cookies were added from the page.
+    #
+    # @since 0.2.2
+    #
+    def from_page(page)
+      cookies = page.cookie_params
+
+      unless cookies.empty?
+        self[page.url.host] = cookies
+        return true
+      end
+
+      return false
+    end
+
+    #
+    # Returns the pre-encoded Cookie for a given host.
+    #
+    # @param [String] host
+    #   The name of the host.
+    #
+    # @return [String]
+    #   The encoded Cookie.
+    #
+    # @since 0.2.2
+    #
+    def for_host(host)
+      if @dirty.include?(host)
+        values = []
+
+        @params[host].each do |name,value|
+          values << "#{name}=#{value}"
+        end
+
+        @cookies[host] = values.join('; ')
+        @dirty.delete(host)
+      end
+
+      return @cookies[host]
+    end
+
+    # 
+    # Clear out the jar, removing all stored cookies.
+    #
+    # @since 0.2.2
+    #
+    def clear!
+      @params.clear
+
+      @dirty.clear
+      @cookies.clear
+      return self
+    end
+
+    #
+    # Size of the current cookie jar store.
+    #
+    # @since 0.2.2
+    #
+    def size
+      @params.size
+    end
+
+    #
+    # Inspects the cookie jar.
+    #
+    # @return [String]
+    #   The inspected version of the cookie jar.
+    #
+    def inspect
+      "#<#{self.class}: #{@params.inspect}>"
+    end
+
+  end
+end

data/lib/spidr/filters.rb

@@ -47,6 +47,8 @@ module Spidr
     #   The patterns which match the URI path extensions to not visit.
     #
     def initialize(options={})
+      super(options)
+
       @schemes = []
 
       if options[:schemes]

data/lib/spidr/page.rb

@@ -1,11 +1,15 @@
 require 'spidr/extensions/uri'
 
+require 'set'
 require 'uri'
 require 'nokogiri'
 
 module Spidr
   class Page
 
+    # Reserved names used within Cookie strings
+    RESERVED_COOKIE_NAMES = Set['path', 'expires', 'domain']
+
     # URL of the page
     attr_reader :url
 
@@ -141,6 +145,18 @@ module Spidr
       @response['Content-Type']
     end
 
+    #
+    # The content types of the page.
+    #
+    # @return [Array<String>]
+    #   The values within the Content-Type header.
+    #
+    # @since 0.2.2
+    #
+    def content_types
+      @headers['content-type']
+    end
+
     #
     # Determines if the page is plain-text.
     #
@@ -148,7 +164,7 @@ module Spidr
     #   Specifies whether the page is plain-text.
     #
     def plain_text?
-      (content_type =~ /text\/plain/) == 0
+      content_types.include?('text/plain')
     end
 
     alias txt? plain_text?
@@ -160,7 +176,7 @@ module Spidr
     #   Specifies whether the page is HTML document.
     #
     def html?
-      (content_type =~ /text\/html/) == 0
+      content_types.include?('text/html')
     end
 
     #
@@ -170,7 +186,7 @@ module Spidr
     #   Specifies whether the page is XML document.
     #
     def xml?
-      (content_type =~ /text\/xml/) == 0
+      content_types.include?('text/xml')
     end
 
     #
@@ -180,7 +196,7 @@ module Spidr
     #   Specifies whether the page is XML Stylesheet (XSL).
     #
     def xsl?
-      (content_type =~ /text\/xsl/) == 0
+      content_types.include?('text/xsl')
     end
 
     #
@@ -190,7 +206,8 @@ module Spidr
     #   Specifies whether the page is JavaScript.
     #
     def javascript?
-      (content_type =~ /(text|application)\/javascript/) == 0
+      content_types.include?('text/javascript') || \
+        content_types.include?('application/javascript')
     end
 
     #
@@ -200,7 +217,7 @@ module Spidr
     #   Specifies whether the page is a CSS stylesheet.
     #
     def css?
-      (content_type =~ /text\/css/) == 0
+      content_types.include?('text/css')
     end
 
     #
@@ -210,7 +227,8 @@ module Spidr
     #   Specifies whether the page is a RSS feed.
     #
     def rss?
-      (content_type =~ /application\/(rss|rdf)\+xml/) == 0
+      content_types.include?('application/rss+xml') || \
+        content_types.include?('application/rdf+xml')
     end
 
     #
@@ -220,7 +238,7 @@ module Spidr
     #   Specifies whether the page is an Atom feed.
     #
     def atom?
-      (content_type =~ /application\/atom\+xml/) == 0
+      content_types.include?('application/atom+xml')
     end
 
     #
@@ -230,7 +248,7 @@ module Spidr
     #   Specifies whether the page is a MS Word document.
     #
     def ms_word?
-      (content_type =~ /application\/msword/) == 0
+      content_types.include?('application/msword')
     end
 
     #
@@ -240,7 +258,7 @@ module Spidr
     #   Specifies whether the page is a PDF document.
     #
     def pdf?
-      (content_type =~ /application\/pdf/) == 0
+      content_types.include?('application/pdf')
     end
 
     #
@@ -250,7 +268,53 @@ module Spidr
     #   Specifies whether the page is a ZIP archive.
     #
     def zip?
-      (content_type =~ /application\/zip/) == 0
+      content_types.include?('application/zip')
+    end
+
+    #
+    # The raw Cookie String sent along with the page.
+    #
+    # @return [String]
+    #   The raw Cookie from the response.
+    #
+    # @since 0.2.2
+    #
+    def cookie
+      (@response['Set-Cookie'] || '')
+    end
+
+    #
+    # The Cookie values sent along with the page.
+    #
+    # @return [Array<String>]
+    #   The Cookies from the response.
+    #
+    # @since 0.2.2
+    #
+    def cookies
+      (@headers['set-cookie'] || [])
+    end
+
+    #
+    # The Cookie key -> value pairs returned with the response.
+    #
+    # @return [Hash{String => String}]
+    #   The cookie keys and values.
+    #
+    # @since 0.2.2
+    #
+    def cookie_params
+      params = {}
+
+      cookies.each do |key_value|
+        key, value = key_value.split('=',2)
+
+        next if RESERVED_COOKIE_NAMES.include?(key)
+
+        params[key] = (value || '')
+      end
+
+      return params
     end
 
     #