pincers 0.6.0 → 0.7.0

data/lib/pincers/{support → http}/cookie.rb

@@ -1,3 +1,3 @@
-module Pincers::Support
+module Pincers::Http
   class Cookie < Struct.new(:name, :value, :domain, :path, :expires, :secure); end
 end

data/lib/pincers/{support → http}/cookie_jar.rb

@@ -1,6 +1,7 @@
-require "pincers/support/cookie"
+require 'pincers/http/cookie'
+require 'pincers/http/utils'
 
-module Pincers::Support
+module Pincers::Http
   class CookieJar
 
     BAD_VALUE_CHARS = /([\x00-\x20\x7F",;\\])/ # RFC 6265 - 4.1.1
@@ -16,7 +17,7 @@ module Pincers::Support
     end
 
     def get(_url, _name)
-      for_origin(URI.parse(_url)).find { |c| c.name == _name }
+      for_origin(Utils.parse_uri(_url)).find { |c| c.name == _name }
     end
 
     def set(_parts)

data/lib/pincers/http/errors.rb

@@ -0,0 +1,26 @@
+module Pincers::Http
+  class RequestError < StandardError
+    extend Forwardable
+
+    def_delegators :@response, :code, :body
+
+    attr_reader :response
+
+    def initialize(_response)
+      @response = _response
+      super _response.message
+    end
+  end
+
+  class EncodingNotSupported < StandardError
+    def initialize(_encoding)
+      super "#{_encoding} is not supported by this operation"
+    end
+  end
+
+  class MaximumRedirectsError < StandardError
+    def initialize
+      super 'Redirection loop detected!'
+    end
+  end
+end

data/lib/pincers/http/request.rb

@@ -0,0 +1,62 @@
+module Pincers::Http
+  class Request
+
+    attr_reader :method, :uri, :headers
+    attr_accessor :data
+
+    def initialize(_method, _uri)
+      @method = _method
+      @uri = _uri
+      @headers = {}
+      @data = nil
+    end
+
+    def url
+      @uri.to_s
+    end
+
+    def native_type
+      case @method
+      when :get then Net::HTTP::Get
+      when :post then Net::HTTP::Post
+      when :put then Net::HTTP::Put
+      when :delete then Net::HTTP::Delete
+      else nil end
+    end
+
+    def set_query(_pairs)
+      _pairs = Utils.hash_to_pairs(_pairs) if _pairs.is_a? Hash
+      @uri.query = Utils.encode_urlencoded(_pairs)
+    end
+
+    def set_form_data(_pairs, _encoding = nil)
+      _pairs = Utils.hash_to_pairs(_pairs) if _pairs.is_a? Hash
+      encoding = default_encoding_for(_pairs)
+      encoding = _encoding if !_encoding.nil? && encoding == Utils::FORM_URLENCODED
+
+      if method == :get
+        raise EncodingNotSupported, encoding if encoding != Utils::FORM_URLENCODED
+        set_query _pairs
+      else
+        headers['Content-Type'] = encoding
+
+        self.data = case encoding
+        when Utils::FORM_URLENCODED
+          Utils.encode_urlencoded _pairs
+        when Utils::FORM_MULTIPART
+          Utils.encode_multipart _pairs
+        else
+          raise Pincers::MissingFeatureError.new "form encoding: #{_encoding}"
+        end
+      end
+    end
+
+  private
+
+    def default_encoding_for(_pairs)
+      has_files  = _pairs.any? { |p| p[1].is_a? IO }
+      has_files ? Utils::FORM_MULTIPART : Utils::FORM_URLENCODED
+    end
+
+  end
+end

data/lib/pincers/http/response_document.rb

@@ -0,0 +1,24 @@
+require 'pincers/http/base_document'
+
+module Pincers::Http
+  class ResponseDocument < BaseDocument
+
+    attr_reader :response
+
+    def initialize(_response)
+      @response = _response
+    end
+
+    def uri
+      @response.uri
+    end
+
+    def content_type
+      @response['Content-Type'] || 'text/plain'
+    end
+
+    def content
+      @response.body
+    end
+  end
+end

data/lib/pincers/http/session.rb

@@ -0,0 +1,99 @@
+require 'pincers/http/errors'
+require 'pincers/http/cookie_jar'
+
+module Pincers::Http
+  class Session
+
+    DEFAULT_HEADERS = {
+      'Accept' => '*/*',
+      'Cache-Control' => 'no-cache'
+    }
+
+    attr_reader :cookie_jar, :headers
+    attr_accessor :proxy_addr, :proxy_port, :redirect_limit
+
+    def initialize(_other = nil)
+      if _other
+        @headers = _other.headers.clone
+        @cookie_jar = _other.cookie_jar.copy
+        @proxy_addr = _other.proxy_addr
+        @proxy_port = _other.proxy_port
+        @redirect_limit = _other.redirect_limit
+      else
+        @headers = DEFAULT_HEADERS
+        @cookie_jar = CookieJar.new
+        @redirect_limit = 10
+      end
+    end
+
+    def proxy=(_value)
+      if _value
+        @proxy_addr, @proxy_port = _value.split ':'
+      else
+        @proxy_addr, @proxy_port = [nil, nil]
+      end
+    end
+
+    def clone
+      self.class.new self
+    end
+
+    def perform(_request)
+      perform_recursive _request, @redirect_limit, nil
+    end
+
+  private
+
+    def perform_recursive(_request, _limit, _redirect)
+      raise MaximumRedirectsError.new if _limit == 0
+
+      uri = _redirect || _request.uri
+      path = uri.request_uri.empty? ? '/' : uri.request_uri
+
+      http_request = _request.native_type.new path
+      http_request.body = _request.data
+
+      copy_headers http_request, @headers
+      copy_headers http_request, _request.headers
+      set_cookies http_request, uri
+
+      http_response = connect(uri).request http_request
+
+      case http_response
+      when Net::HTTPSuccess then
+        update_cookies(uri, http_response)
+        http_response.uri = uri # uri is not always set by net/http
+        http_response
+      when Net::HTTPRedirection then
+        location = Utils.parse_uri(http_response['location'])
+        perform(_request, _limit - 1, location)
+      else
+        handle_error_response http_response
+      end
+    end
+
+    def connect(_uri)
+      conn = Net::HTTP.new _uri.host, _uri.port || 80, @proxy_addr, @proxy_port
+      conn.use_ssl = true if _uri.scheme == 'https'
+      conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      conn
+    end
+
+    def handle_error_response(_http_response)
+      raise RequestError.new _http_response
+    end
+
+    def copy_headers(_request, _headers)
+      _headers.keys.each { |k| _request[k] = _headers[k] }
+    end
+
+    def set_cookies(_request, _uri)
+      _request['Cookie'] = @cookie_jar.for_origin_as_header _uri
+    end
+
+    def update_cookies(_uri, _response)
+      cookies = _response.get_fields('set-cookie')
+      cookies.each { |raw| @cookie_jar.set_raw _uri, raw } if cookies
+    end
+  end
+end

data/lib/pincers/http/utils.rb

@@ -0,0 +1,43 @@
+module Pincers::Http
+  module Utils
+    extend self
+
+    FORM_URLENCODED = 'application/x-www-form-urlencoded'
+    FORM_MULTIPART = 'multipart/form-data'
+
+    def encode_urlencoded(_pairs)
+      _pairs = hash_to_pairs _pairs if _pairs.is_a? Hash
+      _pairs.map { |p| "#{p[0]}=#{CGI.escape(p[1])}" }.join '&'
+    end
+
+    def encode_multipart(_pairs)
+      raise Pincers::MissingFeatureError.new :encode_multipart
+    end
+
+    def hash_to_pairs(_hash)
+      pair_recursive [], _hash
+    end
+
+    def parse_uri(_url)
+      URI.parse _url
+    end
+
+  private
+
+    def pair_recursive(_pairs, _data, _prefix = nil)
+      _data.each do |key, value|
+        key = "#{_prefix}.#{key}" if _prefix
+        case value
+        when Hash
+          pair_recursive _pairs, value, key
+        when Array
+          key = "#{key}[]"
+          value.each { |item| _pairs << [key, item.to_s] }
+        else
+          _pairs << [key.to_s, value.to_s]
+        end
+      end
+      _pairs
+    end
+  end
+end

data/lib/pincers/nokogiri/backend.rb

@@ -0,0 +1,151 @@
+require 'pincers/core/base_backend'
+
+module Pincers::Nokogiri
+  class Backend < Pincers::Core::BaseBackend
+
+    # This is a small bool properties subset, I believe its enough for scrapping.
+    # For information of where to find the full list: http://stackoverflow.com/questions/706384/boolean-html-attributes
+
+    BOOL_PROPERTIES = {
+      checked: [:input_checkbox, :input_radio],
+      selected: [:option],
+      disabled: :all, # no restrictions
+      readonly: [:input_text, :input_password, :textarea],
+      multiple: [:select]
+    }
+
+    def initialize(_document)
+      @document = _document
+    end
+
+    def document
+      @document
+    end
+
+    def document_root
+      [document]
+    end
+
+    def document_title
+      document.title
+    end
+
+    def close_document
+      # no closing needed
+    end
+
+    def search_by_css(_element, _selector, _limit)
+      # nokogiri does not do any query level optimization when searching just one node
+      _element.css _selector
+    end
+
+    def search_by_xpath(_element, _selector, _limit)
+      # nokogiri does not do any query level optimization when searching just one node
+      _element.xpath _selector
+    end
+
+    def extract_element_tag(_element)
+      _element.name
+    end
+
+    def extract_element_text(_element)
+      _element.content
+    end
+
+    def extract_element_html(_element)
+      _element.to_html
+    end
+
+    def extract_element_attribute(_element, _name)
+      _name = _name.to_sym
+      if _name == :value
+        case classify _element
+        when :input_checkbox, :input_radio
+          extract_checkable_value _element
+        when :select
+          extract_select_value _element
+        when :option
+          extract_option_value _element
+        when :textarea
+          _element.content
+        else
+          _element[:value]
+        end
+      elsif is_boolean? _element, _name
+        !_element[_name].nil?
+      else
+        _element[_name]
+      end
+    end
+
+    def set_element_attribute(_element, _name, _value)
+      _name = _name.to_sym
+
+      if _name == :value
+        case classify _element
+        when :select
+          set_select_value _element, _value
+        when :textarea
+          _element.content = _value
+        else
+          _element.set_attribute(_name, _value)
+        end
+      elsif is_boolean? _element, _name
+        set_boolean _element, _name, _value
+      else
+        _element.set_attribute(_name, _value)
+      end
+    end
+
+  private
+
+    def classify(_element)
+      name = _element.name
+      name = "input_#{(_element[:type] || 'text')}" if name == 'input'
+      name = "button_#{(_element[:type] || 'submit')}" if name == 'button'
+      name.to_sym
+    end
+
+    def is_boolean?(_element, _name)
+      permitted = BOOL_PROPERTIES[_name]
+      return false if permitted.nil?
+      return true if permitted == :all
+      return permitted.include? classify(_element)
+    end
+
+    def extract_checkable_value(_element)
+      value = _element[:value]
+      value || 'on'
+    end
+
+    def extract_select_value(_element)
+      multiple = !_element[:multiple].nil?
+      selected = _element.css('option[selected]')
+      if multiple
+        selected.map { |o| extract_option_value(o) }
+      else
+        extract_option_value(selected.first)
+      end
+    end
+
+    def extract_option_value(_element)
+      return nil if _element.nil?
+      _element[:value] || _element.content
+    end
+
+    def set_select_value(_element, _value)
+      _element.xpath(".//option[@selected]").each { |o| set_boolean(o, :selected, false) }
+      to_select = _element.at_xpath(".//option[@value='#{_value}']")
+      to_select = _element.at_xpath(".//option[text()='#{_value}']") if to_select.nil?
+      set_boolean(to_select, :selected, true) unless to_select.nil?
+    end
+
+    def set_boolean(_element, _name, _value)
+      if _value
+        _element.set_attribute(_name, _name)
+      else
+        _element.remove_attribute(_name.to_s)
+      end
+    end
+  end
+end

data/lib/pincers/{factories/nokogiri.rb → nokogiri/factory.rb}

@@ -1,8 +1,8 @@
-require 'pincers/factories/base'
-require 'pincers/backend/nokogiri'
+require "pincers/core/base_factory"
+require "pincers/nokogiri/backend"
 
-module Pincers::Factories
-  class Nokogiri < Base
+module Pincers::Nokogiri
+  class Factory < Pincers::Core::BaseFactory
 
     def load_backend(_options)
       document = _options.delete(:document)
@@ -11,7 +11,7 @@ module Pincers::Factories
         document = ::Nokogiri::HTML document, _options[:url], _options[:encoding], _options[:flags]
       end
 
-      ::Pincers::Backend::Nokogiri.new document
+      Pincers::Nokogiri::Backend.new document
     end
 
   end