glebtv-httpclient 3.1.1 → 3.2.0

data/dist_key/cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID/TCCAuWgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBLMQswCQYDVQQGEwJKUDER
+MA8GA1UECgwIY3Rvci5vcmcxFDASBgNVBAsMC0RldmVsb3BtZW50MRMwEQYDVQQD
+DApodHRwY2xpZW50MB4XDTA5MDUyMTEyMzkwNVoXDTM3MTIzMTIzNTk1OVowSzEL
+MAkGA1UEBhMCSlAxETAPBgNVBAoMCGN0b3Iub3JnMRQwEgYDVQQLDAtEZXZlbG9w
+bWVudDETMBEGA1UEAwwKaHR0cGNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAM2PlkdTH97zvIHoPIMj87wnNvpqIQUD7L/hlysO0XBsmR/XZUeU
+ZKB10JQqMXviWpTnU9KU6xGTx3EI4wfd2dpLwH/d4d7K4LngW1kY7kJlZeJhakno
+GzQ40RSI9WkQ0R9KOE888f7OkTBafcL8UyWFVIMhQBw2d9iNl4Jc69QojayCDoSX
+XbbEP0n8yi7HwIU3RFuX6DtMpOx4/1K7Z002ccOGJ3J9kHgeDQSQtF42cQYC7qj2
+67I/OQgnB7ycxTCP0E7bdXQg+zqsngrhaoNn/+I+CoO7nD4t4uQ+B4agALh4PPxs
+bQD9MCL+VurNGLYv0HVd+ZlLblpddC9PLTsCAwEAAaOB6zCB6DAPBgNVHRMBAf8E
+BTADAQH/MDEGCWCGSAGG+EIBDQQkFiJSdWJ5L09wZW5TU0wgR2VuZXJhdGVkIENl
+cnRpZmljYXRlMB0GA1UdDgQWBBRAnB6XlMoOcm7HVAw+JWxY205PHTAOBgNVHQ8B
+Af8EBAMCAQYwcwYDVR0jBGwwaoAUQJwel5TKDnJux1QMPiVsWNtOTx2hT6RNMEsx
+CzAJBgNVBAYTAkpQMREwDwYDVQQKDAhjdG9yLm9yZzEUMBIGA1UECwwLRGV2ZWxv
+cG1lbnQxEzARBgNVBAMMCmh0dHBjbGllbnSCAQEwDQYJKoZIhvcNAQENBQADggEB
+ABVFepybD5XqsBnOn/oDHvK0xAPMF4Ap4Ht1yMQLObg8paVhANSdqIevPlCr/mPL
+DRjcy+J1fCnE6lCfsfLdTgAjirqt8pm92NccxmJ8hTmMd3LWC1n+eYWaolqTCVRM
+Bpe8UY9enyXrFoudHlr9epr18E6As6VrCSfpXFZkD9WHVSWpzkB3qATu5qcDCzCH
+bI0755Mdm/1hKJCD4l69h3J3OhRIEUPJfHnPvM5wtiyC2dcE9itwE/wdVzBJeIBX
+JQm+Qj+K8qXcRTzZZGIBjw2n46xJgW6YncNCHU/WWfNCYwdkngHS/aN8IbEjhCwf
+viXFisVrDN/+pZZGMf67ZaY=
+-----END CERTIFICATE-----

data/dist_key/gen_dist_cert.rb

@@ -0,0 +1,29 @@
+require 'openssl'
+include OpenSSL
+
+keypair = PKey::RSA.new(File.read("keypair.pem"))
+
+now = Time.now
+cert = X509::Certificate.new
+name = X509::Name.parse("/C=JP/O=ctor.org/OU=Development/CN=httpclient")
+cert.subject = cert.issuer = X509::Name.new(name)
+cert.not_before = now
+cert.not_after = Time.mktime(2038, 1, 1, 8, 59, 59)
+cert.public_key = keypair.public_key
+cert.serial = 0x1
+cert.version = 2 # X509v3
+
+key_usage = ["cRLSign", "keyCertSign"]
+ef = X509::ExtensionFactory.new
+ef.subject_certificate = cert
+ef.issuer_certificate = cert # we needed subjectKeyInfo inside, now we have it
+ext1 = ef.create_extension("basicConstraints","CA:TRUE", true)
+ext2 = ef.create_extension("nsComment","Ruby/OpenSSL Generated Certificate")
+ext3 = ef.create_extension("subjectKeyIdentifier", "hash")
+ext4 = ef.create_extension("keyUsage", key_usage.join(","), true)
+cert.extensions = [ext1, ext2, ext3, ext4]
+ext0 = ef.create_extension("authorityKeyIdentifier", "keyid:always,issuer:always")
+cert.add_extension(ext0)
+cert.sign(keypair, OpenSSL::Digest::SHA512.new)
+
+print cert.to_pem

data/httpclient.gemspec

@@ -0,0 +1,33 @@
+# encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'httpclient/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'glebtv-httpclient'
+  spec.version = HTTPClient::VERSION
+  spec.authors = ['glebtv', 'Hiroshi Nakamura']
+  spec.email = 'glebtv@gmail.com'
+  spec.executables = ['httpclient']
+  spec.homepage = 'http://github.com/glebtv/httpclient'
+  spec.platform = Gem::Platform::RUBY
+  spec.summary = 'Fork of httpclient with some fixes and patches I needed. Please use original gem instead'
+  
+  spec.license = 'ruby'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "PriorityQueue"
+  spec.add_dependency "addressable"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rdoc"
+  spec.add_development_dependency "coveralls"
+end

data/lib/httpclient.rb

@@ -18,6 +18,12 @@ require 'httpclient/session'
 require 'httpclient/http'
 require 'httpclient/auth'
 require 'httpclient/cookie'
+require 'httpclient/cookie'
+
+require 'httpclient/lru_cache'
+require 'httpclient/lru_threadsafe'
+
+require 'active_support/core_ext'
 
 # :main:HTTPClient
 # The HTTPClient class provides several methods for accessing Web resources
@@ -74,7 +80,7 @@ require 'httpclient/cookie'
 #
 # === Invoking other HTTP methods
 #
-# See head, get, post, put, delete, options, propfind, proppatch and trace.
+# See head, get, post, put, patch, delete, options, propfind, proppatch and trace.
 # It returns a HTTP::Message instance as a response.
 #
 # 1. Do HEAD request.
@@ -229,6 +235,13 @@ require 'httpclient/cookie'
 #   ruby -rhttpclient -e 'p HTTPClient.head(ARGV.shift).header["last-modified"]' http://dev.ctor.org/
 #
 class HTTPClient
+  @@dns_cache = HTTPClient::ThreadSafeCache.new(ttl: 20.minutes, soft_ttl: 10.minute, retry_delay: 5.minutes)
+  cattr_accessor :dns_cache
+
+  def own_methods
+    (methods - (self.class.ancestors - [self.class]).collect { |k| k.instance_methods }.flatten).sort
+  end
+  
   RUBY_VERSION_STRING = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE})"
   LIB_NAME = "(#{VERSION}, #{RUBY_VERSION_STRING})"
 
@@ -358,7 +371,7 @@ class HTTPClient
   PROPFIND_DEFAULT_EXTHEADER = { 'Depth' => '0' }
 
   # Default User-Agent header
-  DEFAULT_AGENT_NAME = 'HTTPClient/1.0'
+  DEFAULT_AGENT_NAME = "HTTPClient #{VERSION}"
 
   # Creates a HTTPClient instance which manages sessions, cookies, etc.
   #
@@ -661,7 +674,9 @@ class HTTPClient
       warn("'The field value consists of a single absolute URI' in HTTP spec")
     end
     if https?(uri) && !https?(newuri)
-      raise BadResponseError.new("redirecting to non-https resource")
+      #raise BadResponseError.new("redirecting to non-https resource")
+      # allow redirect to non-https but warn
+      warn("redirecting to non-https resource")
     end
     puts "redirect to: #{newuri}" if $DEBUG
     newuri
@@ -689,6 +704,11 @@ class HTTPClient
     request(:put, uri, argument_to_hash(args, :body, :header), &block)
   end
 
+  # Sends PATCH request to the specified URL.  See request for arguments.
+  def patch(uri, *args, &block)
+    request(:patch, uri, argument_to_hash(args, :body, :header), &block)
+  end
+
   # Sends DELETE request to the specified URL.  See request for arguments.
   def delete(uri, *args, &block)
     request(:delete, uri, argument_to_hash(args, :body, :header), &block)
@@ -713,7 +733,16 @@ class HTTPClient
   def trace(uri, *args, &block)
     request('TRACE', uri, argument_to_hash(args, :query, :header), &block)
   end
-
+  
+  def download_file(uri, file, *args)
+    io = File.open(file, 'a+b')
+    request(:get, uri, argument_to_hash(args, :query, :header, :follow_redirect)) do |s|
+      io.write(s)
+    end
+    io.flush
+    io
+  end
+  
   # Sends a request to the specified URL.
   #
   # method:: HTTP method to be sent.  method.to_s.upcase is used.
@@ -753,8 +782,8 @@ class HTTPClient
   # respond to :read. Bear in mind that some server application does not support
   # chunked request.  At least cgi.rb does not support it.
   def request(method, uri, *args, &block)
-    query, body, header, follow_redirect = keyword_argument(args, :query, :body, :header, :follow_redirect)
-    if [:post, :put].include?(method)
+    query, body, header, is_follow_redirect, filter_block = keyword_argument(args, :query, :body, :header, :follow_redirect, :filter_block)
+    if [:post, :put, :patch].include?(method)
       body ||= ''
     end
     if method == :propfind
@@ -764,12 +793,16 @@ class HTTPClient
     end
     uri = urify(uri)
     if block
-      filtered_block = proc { |res, str|
-        block.call(str)
-      }
+      if filter_block === false
+        filtered_block = block
+      else
+        filtered_block = proc { |res, str|
+          block.call(str)
+        }
+      end
     end
-    if follow_redirect
-      follow_redirect(method, uri, query, body, header, &block)
+    if is_follow_redirect
+      follow_redirect(method, uri, query, body, header, filter_block, &block)
     else
       do_request(method, uri, query, body, header, &filtered_block)
     end
@@ -803,6 +836,13 @@ class HTTPClient
     request_async(:put, uri, nil, body || '', header || {})
   end
 
+  # Sends PATCH request in async style.  See request_async for arguments.
+  # It immediately returns a HTTPClient::Connection instance as a result.
+  def patch_async(uri, *args)
+    body, header = keyword_argument(args, :body, :header)
+    request_async(:patch, uri, nil, body || '', header || {})
+  end
+
   # Sends DELETE request in async style.  See request_async for arguments.
   # It immediately returns a HTTPClient::Connection instance as a result.
   def delete_async(uri, *args)
@@ -943,12 +983,16 @@ private
     ENV[name.downcase] || ENV[name.upcase]
   end
 
-  def follow_redirect(method, uri, query, body, header, &block)
+  def follow_redirect(method, uri, query, body, header, filter_block = true, &block)
     uri = urify(uri)
     if block
-      filtered_block = proc { |r, str|
-        block.call(str) if r.ok?
-      }
+      if filter_block
+        filtered_block = proc { |r, str|
+          block.call(str) if r.ok?
+        }
+      else
+        filtered_block = block
+      end
     end
     if HTTP::Message.file?(body)
       pos = body.pos rescue nil

data/lib/httpclient/lru_cache.rb

@@ -0,0 +1,171 @@
+# for DNS cache
+# We do it manually because Ruby won't do it for us
+# Taken from: https://github.com/kindkid/lrucache/blob/master/lib/lrucache.rb
+# (MIT Licensed). Thanks!
+
+require "priority_queue"
+
+# Not thread-safe!
+class HTTPClient
+  class LRUCache
+
+    attr_reader :default, :max_size, :ttl, :soft_ttl, :retry_delay
+
+    def initialize(opts={})
+      @max_size = Integer(opts[:max_size] || 100)
+      @default = opts[:default]
+      @eviction_handler = opts[:eviction_handler]
+      @ttl = Float(opts[:ttl] || 0)
+      @soft_ttl = Float(opts[:soft_ttl] || 0)
+      @retry_delay = Float(opts[:retry_delay] || 0)
+      raise "max_size must not be negative" if @max_size < 0
+      raise "ttl must not be negative" if @ttl < 0
+      raise "soft_ttl must not be negative" if @soft_ttl < 0
+      raise "retry_delay must not be negative" if @retry_delay < 0
+
+      @pqueue = PriorityQueue.new
+      @data = {}
+      @counter = 0
+    end
+
+    def clear
+      @data.clear
+      @pqueue.delete_min until @pqueue.empty?
+      @counter = 0 #might as well
+    end
+
+    def include?(key)
+      datum = @data[key]
+      return false if datum.nil?
+      if datum.expired?
+        delete(key)
+        false
+      else
+        access(key)
+        true
+      end
+    end
+
+    def store(key, value, args={})
+      evict_lru! unless @data.include?(key) || @data.size < max_size
+      ttl, soft_ttl, retry_delay = extract_arguments(args)
+      expiration = expiration_date(ttl)
+      soft_expiration = expiration_date(soft_ttl)
+      @data[key] = Datum.new(value, expiration, soft_expiration)
+      access(key)
+      value
+    end
+
+    alias :[]= :store
+
+    def fetch(key, args={})
+      datum = @data[key]
+      if datum.nil?
+        if block_given?
+          store(key, value = yield, args)
+        else
+          @default
+        end
+      elsif datum.expired?
+        delete(key)
+        if block_given?
+          store(key, value = yield, args)
+        else
+          @default
+        end
+      elsif datum.soft_expired?
+        if block_given?
+          begin
+            store(key, value = yield, args)
+          rescue RuntimeError => e
+            access(key)
+            ttl, soft_ttl, retry_delay = extract_arguments(args)
+            datum.soft_expiration = (Time.now + retry_delay) if retry_delay > 0
+            datum.value
+          end
+        else
+          access(key)
+          datum.value
+        end
+      else
+        access(key)
+        datum.value
+      end
+    end
+
+    alias :[] :fetch
+
+    def empty?
+      size == 0
+    end
+
+    def size
+      @data.size
+    end
+
+    def keys
+      @data.keys
+    end
+
+    def delete(key)
+      @pqueue.delete(key)
+      datum = @data.delete(key)
+      datum.value unless datum.nil?
+    end
+
+    private
+
+    class Datum
+      attr_reader :value, :expiration, :soft_expiration
+      attr_writer :soft_expiration
+      def initialize(value, expiration, soft_expiration)
+        @value = value
+        @expiration = expiration
+        @soft_expiration = soft_expiration
+      end
+
+      def expired?
+        !@expiration.nil? && @expiration <= Time.now
+      end
+
+      def soft_expired?
+        !@soft_expiration.nil? && @soft_expiration <= Time.now
+      end
+    end
+
+    def expiration_date(ttl)
+      if ttl.is_a?(Time)
+        ttl
+      else
+        ttl = Float(ttl)
+        (ttl > 0) ? (Time.now + ttl) : nil
+      end
+    end
+
+    def extract_arguments(args)
+      if args.is_a?(Hash)
+        ttl = args[:ttl] || @ttl
+        soft_ttl = args[:soft_ttl] || @soft_ttl
+        retry_delay = args[:retry_delay] || @retry_delay
+        [ttl, soft_ttl, retry_delay]
+      else
+        # legacy arg
+        ttl = args || @ttl
+        [ttl, @soft_ttl, @retry_delay]
+      end
+    end
+
+    def evict_lru!
+      key, priority = @pqueue.delete_min
+      unless priority.nil?
+        datum = @data.delete(key)
+        @eviction_handler.call(datum.value) if @eviction_handler && datum
+      end
+    end
+
+    def access(key)
+      @pqueue.change_priority(key, @counter += 1)
+    end
+
+  end
+end

data/lib/httpclient/lru_threadsafe.rb

@@ -0,0 +1,29 @@
+# Make LRU cache Threadsafe
+# Idea from: https://github.com/SamSaffron/lru_redux/blob/master/lib/lru_redux/thread_safe_cache.rb
+# (MIT License)
+
+require 'thread'
+require 'monitor'
+
+class HTTPClient::ThreadSafeCache < HTTPClient::LRUCache
+  include MonitorMixin
+
+  def initialize(opts={})
+    super(opts)
+  end
+
+  def self.synchronize(*methods)
+    methods.each do |method|
+      define_method method do |*args, &blk|
+        synchronize do
+          super(*args, &blk)
+        end
+      end
+    end
+
+    alias :[]= :store
+    alias :[] :fetch
+  end
+
+  synchronize :fetch, :store, :delete, :clear, :include?, :empty, :size, :keys 
+end

data/lib/httpclient/session.rb

@@ -11,21 +11,19 @@
 # I asked Maebashi-san he agreed that I can redistribute it under the same terms
 # of Ruby.  Many thanks to Maebashi-san.
 
-
+require 'ipaddr'
 require 'socket'
 require 'thread'
 require 'stringio'
 require 'zlib'
+require 'timeout'
+require 'resolv'
 
-require 'httpclient/timeout'
 require 'httpclient/ssl_config'
 require 'httpclient/http'
 require 'httpclient/util'
 
-
 class HTTPClient
-
-
   # Represents a Site: protocol scheme, host String and port Number.
   class Site
     # Protocol scheme.
@@ -157,6 +155,13 @@ class HTTPClient
         @proxy = Site.new(proxy)
       end
     end
+    
+    def timeout(time, exn = nil, &block)
+      return yield if time.nil? || time.zero?
+      Timeout.timeout(time, exn) do
+        yield
+       end
+    end
 
     def query(req, via_proxy)
       req.http_body.chunk_size = @chunk_size
@@ -510,7 +515,13 @@ class HTTPClient
           str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
           @debug_dev << HexDump.encode(str).join("\n")
         else
-          @debug_dev << str
+          begin
+            @debug_dev << str
+          rescue
+            require 'hexdump'
+            str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+            @debug_dev << HexDump.encode(str).join("\n")
+          end
         end
       end
     end
@@ -535,7 +546,6 @@ class HTTPClient
 
   # Manages a HTTP session with a Site.
   class Session
-    include HTTPClient::Timeout
     include Util
 
     # Destination site
@@ -736,7 +746,7 @@ class HTTPClient
         end
       end
       if @agent_name && req.header.get('User-Agent').empty?
-        req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
+        req.header.set('User-Agent', "#{@agent_name}")
       end
       if @from && req.header.get('From').empty?
         req.header.set('From', @from)
@@ -806,12 +816,32 @@ class HTTPClient
       socket = nil
       begin
         @debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
+        clean_host = site.host.delete("[]")
+        clean_local = @socket_local.host.delete("[]")
         if str = @test_loopback_http_response.shift
-          socket = LoopBackSocket.new(site.host, site.port, str)
-        elsif @socket_local == Site::EMPTY
-          socket = TCPSocket.new(site.host, site.port)
+          socket = LoopBackSocket.new(clean_host, site.port, str)
         else
-          socket = TCPSocket.new(site.host, site.port, @socket_local.host, @socket_local.port)
+          begin
+            ip = IPAddr.new(clean_host).to_s
+            # puts "! #{site.host} IS AN IP!\n"
+            @debug_dev <<  "! #{site.host} IS AN IP!\n" if @debug_dev
+          rescue
+            ip = HTTPClient.dns_cache.fetch clean_host do
+              Timeout.timeout(10) do
+                @debug_dev << "! RESOLVING #{clean_host}" if @debug_dev
+                ip = Resolv.getaddress(clean_host)
+                # puts "! RESOLVED #{clean_host} TO #{ip}\n"
+                @debug_dev << "! RESOLVED #{clean_host} TO #{ip}\n" if @debug_dev
+                ip
+              end
+            end
+          end
+
+          if @socket_local == Site::EMPTY
+            socket = TCPSocket.new(ip, site.port)
+          else
+            socket = TCPSocket.new(ip, site.port, clean_local, @socket_local.port)
+          end
         end
         if @debug_dev
           @debug_dev << "! CONNECTION ESTABLISHED\n"
@@ -957,8 +987,9 @@ class HTTPClient
       return nil if @content_length == 0
       while true
         buf = HTTPClient::Util.get_buf
+
         maxbytes = @read_block_size
-        maxbytes = @content_length if maxbytes > @content_length
+        maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
         timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(maxbytes, buf)
@@ -980,6 +1011,7 @@ class HTTPClient
     RS = "\r\n"
     def read_body_chunked(&block)
       buf = HTTPClient::Util.get_buf
+
       while true
         len = @socket.gets(RS)
          if len.nil? # EOF
@@ -1009,6 +1041,7 @@ class HTTPClient
       end
       while true
         buf = HTTPClient::Util.get_buf
+
         timeout(@receive_timeout, ReceiveTimeoutError) do
           begin
             @socket.readpartial(@read_block_size, buf)
@@ -1023,6 +1056,12 @@ class HTTPClient
         end
       end
     end
+
+    def empty_bin_str
+      str = ''
+      str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+      str
+    end
   end