pincers 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb985ceb492cc98c2bfa31c449d2910956a598ce
-  data.tar.gz: d87917a61c5e611f2e2a83e03ffbe7821b94e127
+  metadata.gz: 51cc155a36f380781aa91b00a53f5996aae5b9b1
+  data.tar.gz: b7ef59a62aa573b6be45914692fc603ef91195c0
 SHA512:
-  metadata.gz: 4175ea99d6f5e46ca5bc742a4d9a4884991d8a92ccca0d5dbf95e86dbcbdea3b6c327cb784557ec2ccd51cd67189093d935efa56bfa9edba776a4ff15a3b76a9
-  data.tar.gz: 659b15f7fbe7570a1f78eaadc2af572a78fb68e00496e789102732dc935ecdcd317a14cbfb07d6680021463146e67d1e9cab2eaafb39dd90db66e00a727cc08c
+  metadata.gz: 78e3f46f2d2a4bd2bcb82d6ca2c5eda70f2683f5bc769eb086204eb82e7168b7b02a6687550b47870187f2172319184d6f47d1b98a33984f7fc50860f40a0868
+  data.tar.gz: 95d1bbf4b2cc3ccea044ecc0c8bfea63477b56d489487edf2d9f110943e5b231d955f25c47e25a0e8c661a593d696c9003642cacdceaace35ad26f1ae7943c5b

data/lib/pincers/backend/base.rb

@@ -120,6 +120,10 @@ module Pincers::Backend
       ensure_implementation :switch_to_parent_frame
     end
 
+    def fetch_resource
+      ensure_implementation :fetch_resource
+    end
+
   private
 
     def ensure_implementation(_name)

data/lib/pincers/backend/webdriver.rb

@@ -1,5 +1,7 @@
-require 'selenium-webdriver'
-require 'pincers/backend/base'
+require "selenium-webdriver"
+require "pincers/backend/base"
+require "pincers/support/http_client"
+require "pincers/core/download"
 
 module Pincers::Backend
 
@@ -149,6 +151,12 @@ module Pincers::Backend
       driver.switch_to.default_content
     end
 
+    def fetch_resource(_url)
+      url = URI::join(driver.current_url, _url)
+      response = as_http_client.get url
+      Pincers::Core::Download.from_http_response response
+    end
+
   private
 
     def search(_element, _query, _limit)
@@ -192,6 +200,31 @@ module Pincers::Backend
       _element
     end
 
+    def as_http_client
+      Pincers::Support::HttpClient.new({
+        proxy: proxy_address,
+        cookies: cookie_jar,
+        headers: {
+          'User-Agent' => user_agent
+        }
+      })
+    end
+
+    def user_agent
+      driver.execute_script("return navigator.userAgent;")
+    end
+
+    def proxy_address
+      proxy = driver.capabilities.proxy
+      proxy.nil? ? nil : (proxy.http || proxy.ssl)
+    end
+
+    def cookie_jar
+      jar = Pincers::Support::CookieJar.new
+      driver.manage.all_cookies.each { |c| jar.set c }
+      jar
+    end
+
   end
 
 end

data/lib/pincers/core/cookies.rb

@@ -0,0 +1,16 @@
+require 'ostruct'
+
+module Pincers::Core
+  class Cookies
+    include Enumerable
+
+    def initialize(_backend)
+      @backend = _backend
+    end
+
+    def each
+      @backend.fetch_cookies.each { |c| yield OpenStruct.new c }
+    end
+
+  end
+end

data/lib/pincers/core/download.rb

@@ -0,0 +1,13 @@
+module Pincers::Core
+  class Download < Struct.new(:mime, :data)
+
+    def self.from_http_response(_response)
+      self.new _response['Content-Type'] || 'text/plain', _response.body
+    end
+
+    def store(_path)
+      File.write _path, data
+    end
+
+  end
+end

data/lib/pincers/core/root_context.rb

@@ -1,4 +1,4 @@
-require 'pincers/support/cookie_jar'
+require 'pincers/core/cookies'
 require 'pincers/core/search_context'
 
 module Pincers::Core
@@ -41,7 +41,7 @@ module Pincers::Core
     end
 
     def cookies
-      @cookies ||= CookieJar.new backend
+      @cookies ||= Cookies.new backend
     end
 
     def goto(_urlOrOptions)
@@ -61,6 +61,10 @@ module Pincers::Core
       self
     end
 
+    def download(_url)
+      wrap_errors { backend.fetch_resource _url }
+    end
+
     def forward(_steps=1)
       wrap_errors { backend.navigate_forward _steps }
       self

data/lib/pincers/extension/queries.rb

@@ -58,5 +58,11 @@ module Pincers::Extension
       type.to_sym
     end
 
+    def download
+      url = attribute(:href) || attribute(:src)
+      raise Pincers::NavigationError.new(self, 'No resource url was found') if url.nil?
+      root.download(attribute(:href) || attribute(:src))
+    end
+
   end
 end

data/lib/pincers/support/cookie.rb

@@ -0,0 +1,3 @@
+module Pincers::Support
+  class Cookie < Struct.new(:name, :value, :domain, :path, :expires, :secure); end
+end

data/lib/pincers/support/cookie_jar.rb

@@ -1,15 +1,161 @@
-require 'ostruct'
+require "pincers/support/cookie"
 
 module Pincers::Support
   class CookieJar
-    include Enumerable
 
-    def initialize(_backend)
-      @backend = _backend
+    BAD_VALUE_CHARS = /([\x00-\x20\x7F",;\\])/ # RFC 6265 - 4.1.1
+
+    attr_reader :cookies
+
+    def initialize(_cookies=nil)
+      @cookies = _cookies || []
+    end
+
+    def copy
+      self.class.new @cookies.clone
+    end
+
+    def get(_url, _name)
+      for_origin(URI.parse(_url)).find { |c| c.name == _name }
+    end
+
+    def set(_parts)
+      return nil unless _parts[:domain]
+      return nil unless _parts[:name]
+      return nil unless _parts[:value]
+
+      cookie = Cookie.new(
+        _parts[:name],
+        _parts[:value],
+        _parts[:domain].gsub(/^\./,''),
+        _parts[:path] || '/',
+        _parts[:expires],
+        _parts[:secure]
+      )
+
+      replace_cookie cookie
+      cookie
+    end
+
+    def set_raw(_uri, _raw)
+      cookie = decode_cookie _raw
+
+      cookie.domain = compute_domain cookie.domain, _uri.host
+      cookie.path = compute_path cookie.path, _uri.path
+
+      return nil if cookie.domain.nil? or cookie.path.nil?
+
+      replace_cookie(cookie)
+      cookie
+    end
+
+    def for_origin(_uri)
+      @cookies.select do |c|
+        domains_match c.domain, _uri.host and paths_match c.path, _uri.path
+      end
+    end
+
+    def for_origin_as_header(_uri)
+      for_origin(_uri).map { |c| "#{c.name}=#{quote(c.value)}" }.join('; ')
+    end
+
+  private
+
+    def decode_cookie(_raw)
+      # taken from WEBrick implementation
+      cookie_elem = _raw.split(/;/)
+      first_elem = cookie_elem.shift
+      first_elem.strip!
+      key, value = first_elem.split(/\=/, 2)
+
+      cookie = Cookie.new(key, dequote(value))
+      cookie_elem.each do |pair|
+        pair.strip!
+        key, value = pair.split(/\=/, 2)
+        value = dequote(value.strip) if value
+
+        case key.downcase
+        when "domain"  then cookie.domain  = value.downcase
+        when "path"    then cookie.path    = value.downcase
+        when "expires" then cookie.expires = value
+        # when "max-age" then cookie.max_age = Integer(value)
+        # when "comment" then cookie.comment = value
+        # when "version" then cookie.version = Integer(value)
+        when "secure"  then cookie.secure = true
+        end
+      end
+
+      cookie
+    end
+
+    def domains_match(_cookie_domain, _request_domain)
+      # RFC 6265 - 5.1.3
+      # TODO: ensure request domain is not an IP
+      return true if _cookie_domain == _request_domain
+      return true if _request_domain.end_with? ".#{_cookie_domain}"
+      return false
+    end
+
+    def paths_match(_cookie_path, _request_path)
+      # RFC 6265 - 5.1.4
+      _request_path = '/' if _request_path.empty?
+      return true if _cookie_path == _request_path
+      return true if _cookie_path[-1] == '/' and _request_path.start_with? _cookie_path
+      return true if _request_path.start_with? "#{_cookie_path}/"
+      return false
+    end
+
+    def compute_domain(_cookie_domain, _request_domain)
+      return _request_domain if _cookie_domain.nil?
+      # cookies with different domain are discarded
+      return nil unless _cookie_domain.end_with? _request_domain
+      return _cookie_domain.gsub(/^\./,'') # remove leading dot
+    end
+
+    def compute_path(_cookie_path, _request_path)
+      default_path = compute_default_path(_request_path)
+      return default_path if _cookie_path.nil?
+      return nil unless _cookie_path.start_with? default_path
+      return _cookie_path
+    end
+
+    def compute_default_path(_request_path)
+      # RFC 6265 - 5.1.4
+      return '/' unless _request_path[0] === '/'
+      ls_idx = _request_path.rindex('/')
+      return '/' unless ls_idx > 0
+      _request_path[0..ls_idx]
+    end
+
+    def replace_cookie(_cookie)
+      @cookies.each_with_index do |cookie, i|
+        if equivalent(cookie, _cookie)
+          @cookies[i] = _cookie
+          return
+        end
+      end
+
+      @cookies << _cookie
+    end
+
+    def dequote(_str)
+      # taken from WEBrick implementation
+      ret = (/\A"(.*)"\Z/ =~ _str) ? $1 : _str.dup
+      ret.gsub!(/\\(.)/, "\\1")
+      ret
+    end
+
+    def quote(_str)
+      # taken from WEBrick implementation and the http-cookie gem
+      return _str unless BAD_VALUE_CHARS === _str
+      '"' << _str.gsub(/[\\\"]/o, "\\\1") << '"'
     end
 
-    def each
-      @backend.fetch_cookies.each { |c| yield OpenStruct.new c }
+    def equivalent(_cookie_a, _cookie_b)
+      return false unless _cookie_a.domain == _cookie_b.domain
+      return false unless _cookie_a.path == _cookie_b.path
+      return false unless _cookie_a.name == _cookie_b.name
+      return true
     end
 
   end

data/lib/pincers/support/http_client.rb

@@ -0,0 +1,123 @@
+require "uri"
+require "pincers/support/cookie_jar"
+
+module Pincers::Support
+  class HttpClient
+
+    class HttpRequestError < StandardError
+      extend Forwardable
+
+      def_delegators :@response, :code, :body
+
+      attr_reader :response
+
+      def initialize(_response)
+        @response = _response
+        super _response.message
+      end
+    end
+
+    class MaximumRedirectsError < StandardError
+      def initialize
+        super 'Redirection loop detected!'
+      end
+    end
+
+    attr_reader :proxy_addr, :proxy_port, :cookies
+
+    def initialize(_options={})
+      if _options[:proxy]
+        @proxy_addr, @proxy_port = _options[:proxy].split ':'
+      end
+
+      @cookies = if _options[:cookies]
+        _options[:cookies].copy
+      else
+        CookieJar.new
+      end
+
+      @default_headers = _options[:headers]
+    end
+
+    def get(_url, _query={}, _headers={})
+      # TODO: append query string?
+      perform_request Net::HTTP::Get, URI(_url), _headers
+    end
+
+    def post(_url, _data, _headers={})
+      perform_request Net::HTTP::Post, URI(_url), _headers do |req|
+        req.body = prepare_data(_data)
+      end
+    end
+
+    def put(_url, _data, _headers={})
+      perform_request Net::HTTP::Put, URI(_url), _headers do |req|
+        req.body = prepare_data(_data)
+      end
+    end
+
+    def delete(_url)
+      perform_request Net::HTTP::Delete, URI(_url), _headers
+    end
+
+  private
+
+    def perform_request(_req_type, _uri, _headers, _limit=10)
+
+      raise MaximumRedirectsError.new if _limit == 0
+
+      request = _req_type.new(_uri.request_uri.empty? ? '/' : _uri.request_uri)
+      build_headers(request, _headers)
+      set_cookies(request, _uri)
+      yield request if block_given?
+
+      response = build_client(_uri).request request
+
+      case response
+      when Net::HTTPSuccess then
+        update_cookies(_uri, response)
+        response
+      when Net::HTTPRedirection then
+        location = response['location']
+        perform_request(_req_type, URI.parse(location), _headers, _limit - 1)
+      else
+        handle_error_response response
+      end
+    end
+
+    def build_client(uri)
+      client = Net::HTTP.new uri.host, uri.port || 80, proxy_addr, proxy_port
+      client.use_ssl = true if uri.scheme == 'https'
+      client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      client
+    end
+
+    def handle_error_response(_response)
+      raise HttpRequestError.new _response
+    end
+
+    def prepare_data(_data)
+      if _data.is_a? Hash
+        _data.keys.map { |k| "#{k}=#{_data[k]}" }.join '&'
+      else _data end
+    end
+
+    def build_headers(_request, _headers)
+      copy_headers _request, @default_headers if @default_headers
+      copy_headers _request, _headers
+    end
+
+    def set_cookies(_request, _uri)
+      _request['Cookie'] = @cookies.for_origin_as_header _uri
+    end
+
+    def update_cookies(_uri, _response)
+      cookies = _response.get_fields('set-cookie')
+      cookies.each { |raw| @cookies.set_raw _uri, raw } if cookies
+    end
+
+    def copy_headers(_request, _headers)
+      _headers.keys.each { |k| _request[k] = _headers[k] }
+    end
+  end
+end

data/lib/pincers/version.rb

@@ -1,3 +1,3 @@
 module Pincers
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pincers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-03 00:00:00.000000000 Z
+date: 2015-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -217,6 +217,8 @@ files:
 - lib/pincers/backend/base.rb
 - lib/pincers/backend/nokogiri.rb
 - lib/pincers/backend/webdriver.rb
+- lib/pincers/core/cookies.rb
+- lib/pincers/core/download.rb
 - lib/pincers/core/root_context.rb
 - lib/pincers/core/search_context.rb
 - lib/pincers/errors.rb
@@ -228,7 +230,9 @@ files:
 - lib/pincers/factories/webdriver.rb
 - lib/pincers/factory.rb
 - lib/pincers/support/configuration.rb
+- lib/pincers/support/cookie.rb
 - lib/pincers/support/cookie_jar.rb
+- lib/pincers/support/http_client.rb
 - lib/pincers/support/query.rb
 - lib/pincers/support/xpath_builder.rb
 - lib/pincers/version.rb