http-request 1.0.5 → 1.1.0

data/{README → README.md}

@@ -1,6 +1,5 @@
-HttpRequest
-===========
+# HttpRequest
 
 Simple HTTP client written in Ruby. Intended to be easier to work with then Net/HTTP.
 
-Test on Ruby 1.8.7 and 1.9.2.
+Tested on Ruby 1.8 and 1.9 (MRI, JRuby).

data/lib/http-request.rb

@@ -1,198 +1,21 @@
-# +-------------------------------------------------------------------------------+
-# | HttpRequest                                                                   |
-# +-------------------------------------------------------------------------------+
-# | Author: Nulayer Inc. / www.nulayer.com                                        |
-# +-------------------------------------------------------------------------------+
-# | BSD LICENSE                                                                   |
-# | Copyright (c) 2007-2011, Nulayer Inc.                                         |
-# | All rights reserved.                                                          |
-# |                                                                               |
-# | Redistribution and use in source and binary forms, with or without            |
-# | modification, are permitted provided that the following conditions are met:   |
-# |     * Redistributions of source code must retain the above copyright          |
-# |       notice, this list of conditions and the following disclaimer.           |
-# |     * Redistributions in binary form must reproduce the above copyright       |
-# |       notice, this list of conditions and the following disclaimer in the     |
-# |       documentation and/or other materials provided with the distribution.    |
-# |     * Neither the name of the <organization> nor the                          |
-# |       names of its contributors may be used to endorse or promote products    |
-# |       derived from this software without specific prior written permission.   |
-# |                                                                               |
-# | THIS SOFTWARE IS PROVIDED BY NuLayer Inc. "AS IS" AND ANY                     |
-# | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED     |
-# | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE        |
-# | DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY            |
-# | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES    |
-# | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;  |
-# | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND   |
-# | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT    |
-# | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
-# | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.                  |
-# +-------------------------------------------------------------------------------+
-
 require 'net/http'
+require 'addressable/uri'
 
-class HttpRequest
-  VERSION = '1.0.5'
-
-  class HTTPError < StandardError; end
-  class UnexpectedError < StandardError; end
-
-  class UnreachableError < HTTPError; end
-  class ConnectionResetError < HTTPError; end
-  class TimeoutError < HTTPError; end
-  class RedirectError < HTTPError; end
-
-  attr_accessor :uri
-  attr_accessor :header
-  attr_accessor :body
-  attr_accessor :status
-
-  class << self
-    attr_accessor :user_agent, :timeout, :max_retries
-  end
-
-
-  def self.head(url, options={})
-    http = HttpRequest.new(url)
-    http.head(options)
-  end
-
-  def self.get(url, options={})
-    http = HttpRequest.new(url)
-    http.get(options)
-  end
-
-  def initialize(url)
-    self.uri = url
-  end
-
-  def head(options={})
-    request(:head, options)
-  end
-
-  def get(options={})
-    request(:get, options)
-  end
-
-  def request(type, options={})
-    user_agent     = options[:user_agent] || HttpRequest.user_agent || 'HttpRequest'
-    timeout        = (options[:timeout] || HttpRequest.timeout || 10).to_i
-    redirect_limit = options[:redirect_limit] || 5
-    output_file    = options[:output_file]
-    content_max    = options[:content_max]
-    max_retries     = HttpRequest.max_retries || 3
+module HttpRequest
+  extend self
 
-    # Check if we're in a redirecting loop
-    if redirect_limit == 0
-      raise RedirectError, "HTTP redirect too deep: #{uri.to_s}"
-    end
-
-    # Prepare the request
-    req_path = uri.path
-    req_path += "?#{uri.query}" if !uri.query.nil?
-
-    req_klass = (type == :head ? Net::HTTP::Head : Net::HTTP::Get)
-    req = req_klass.new(req_path, { 'User-Agent' => user_agent })
-
-    # Basic authentication
-    if uri.user && uri.password
-      req.basic_auth uri.user, uri.password
-    end
-
-    # HTTP request block
-    @retries = max_retries
-    begin
-      http = Net::HTTP.new(uri.host, uri.port)
-
-      http.open_timeout = timeout
-      http.read_timeout = timeout
-
-      http.start do |http|
-        http.request(req) do |response|
-          # Save the header
-          self.header = response.header
-          self.body   = nil
-
-          # Restrict downloading of content
-          download_content = true
-          if !content_max.nil? && header['content-length'].to_i > content_max
-            download_content = false
-          end
-
-          # Get the body
-          if download_content
-            if output_file
-              open(output_file, "w") do |f|
-                response.read_body { |data| f.write(data) }
-              end
-            else
-              self.body = response.read_body
-            end
-          end
-
-          # Handle URL Redirection
-          if response.kind_of?(Net::HTTPRedirection) && !header['location'].nil?
-            update_uri_location(header['location'])
-            request type, options.merge({ :redirect_limit => redirect_limit-1 })
-          end
-        end
-      end
-    rescue ::Errno::ENETUNREACH
-      raise UnreachableError, "network is unreachable: #{uri.to_s}"
-    rescue ::Errno::ECONNRESET, ::Errno::ECONNREFUSED, ::SocketError
-      raise ConnectionResetError, "connection reset by peer: #{uri.to_s}"
-    rescue ::Timeout::Error
-      raise TimeoutError, "connection timed out: #{uri.to_s}"
-    rescue ::Errno::EINTR => ex
-      # Transmission was interrupted, retry the request just once.
-      # This usually occurs because the process received a signal.
-      @retries -= 1
-      if @retries >= 0
-        retry
-      else
-        raise ex
-      end
-    rescue HTTPError => ex
-      # Catch exceptions that may be raised above -- not sure why we have to do this
-      raise ex
-    rescue => ex
-      raise UnexpectedError, "#{ex.inspect} -- #{uri.to_s}"
-    ensure
-      http.finish rescue nil
-      http = nil
-    end
-
-    return self
-  end
-
-  def header=(header)
-    return if header.nil?
-    @header     = header
-    self.status = header.code.to_i
-  end
-
-  def reset!
-    self.header = nil
-    self.body   = nil
-    self.status = nil
+  def pool(key)
+    Client.pool(key)
   end
 
-  def uri=(url)
-    @uri = URI.parse(url)
-    @uri.path = '/' if @uri.path.empty?
-    reset!
-  end
-
-  def update_uri_location(location)
-    if header['location'].to_s[0].to_s.chr == '/'
-      url = "#{uri.scheme}://#{uri.host}"
-      url += ":#{uri.port}" if uri.port != 80 && uri.port != 443
-      url += location
-    else
-      url = location
-    end
-
-    self.uri = url
+  def open(url, options={})
+    client = Client.new(options)
+    client.open(url)
+    client
   end
 end
+
+require 'http-request/version'
+require 'http-request/client'
+require 'http-request/response'
+require 'http-request/connection'

data/lib/http-request/client.rb

@@ -0,0 +1,71 @@
+module HttpRequest
+  class Client
+    attr_reader :connections, :options
+
+    # Syntax sugar to manage clients in the current thread
+    def self.pool(key)
+      key = "http-request:#{key}"
+      Thread.current[key] ||= Client.new
+    end
+
+    def self.delete_from_pool(key)
+      Thread.current[key] = nil
+    end
+
+    def initialize(options={})
+      @options = options # Connection options
+      @connections = []
+    end
+
+    def open(url)
+      # Let's close any dead connections as we open new ones
+      cleanup_dead_connections!
+
+      # Reuse existing connection or open a new one..
+      conn = connection(url)
+      if conn.nil? || conn.closed?
+        conn = Connection.new(url, options)
+        connections << conn
+      end
+      conn
+    end
+
+    def close(url=nil)
+      if !url.nil?
+        conn = connection(url)
+        conn.close
+        connections.delete(conn)
+      else
+        connections.each {|conn| conn.close }
+        @connections = []
+      end
+    end
+
+    [:get, :post, :put, :delete, :head].each do |verb|
+      define_method(verb) do |*args|
+        url, headers, body = args
+        request(verb, url, headers || {}, body)
+      end
+    end
+
+    def request(method, url, headers={}, body=nil)
+      conn = open(url)
+      conn.send method, url, headers, body
+    end
+
+    def connection(url)
+      connections.find {|conn| conn.server_id == server_id(url) }
+    end
+
+    def server_id(url)
+      Addressable::URI.parse(url).origin
+    end
+
+    private
+
+      def cleanup_dead_connections!
+        dead_conns = connections.collect {|conn| conn if conn.closed? }.compact
+        dead_conns.each {|conn| @connections.delete(conn) }
+      end
+  end
+end

data/lib/http-request/connection.rb

@@ -0,0 +1,119 @@
+module HttpRequest
+  class Connection
+    attr_reader :uri, :http
+    attr_accessor :user_agent, :max_retries, :timeout, :auto_redirect, :keep_alive
+
+    def initialize(url, options={})
+      # Defaults
+      @user_agent     = options[:user_agent] || "HttpRequest v#{HttpRequest::VERSION}"
+      @max_retries    = options[:max_retries] || 3
+      @timeout        = options[:timeout] || 30 # 30 second timeout by default
+      @auto_redirect  = options[:auto_redirect].nil? ? true : options[:auto_redirect]
+      @keep_alive     = options[:keep_alive].nil? ? true : options[:keep_alive]
+
+      # Let's do this
+      @uri = Addressable::URI.parse(url)
+      open
+    end
+
+    def open
+      @http = Net::HTTP.new(uri.host, uri.port)
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+      http.start
+
+      if Socket.const_defined?(:TCP_NODELAY)
+        socket.io.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+      end
+    end
+
+    def close
+      http.finish if http
+    end
+
+    def opened?
+      !closed?
+    end
+
+    def closed?
+      !(!http.nil? && http.started?)
+    end
+
+    def socket
+      return if http.nil?
+      http.instance_variable_get(:@socket)
+    end
+
+    def server_id
+      uri.origin
+    end
+
+    [:get, :post, :put, :delete, :head].each do |verb|
+      define_method(verb) do |*args|
+        path, headers, body = args
+        request(verb, path, headers || {}, body)
+      end
+    end
+
+    def request(verb, path, headers={}, body=nil)
+      if path =~ /^http/i
+        new_uri = Addressable::URI.parse(path)
+        if new_uri.origin != uri.origin
+          # TODO.. get an error class for this..
+          raise "Ummm... you're trying to talk to another server you duffis.."
+        end
+        uri.path = new_uri.path
+      end
+
+      # Open the connection if its closed or hasn't been started..
+      open if closed?
+
+      # headers['Date'] = Time.now.httpdate # need this..?
+      headers['User-Agent'] ||= user_agent
+      headers['Connection'] ||= 'keep-alive' if keep_alive
+      headers['Content-Length'] ||= body.bytesize.to_s if body
+
+      # TODO.. link traversal..
+      # ..will our host change tho... like the thing we actually connect to?
+      # it could very well... so we should check that out..
+      # maybe we have a "host" or "host_id" variable for this stuff..?
+
+      # Build request
+      req_klass = instance_eval("Net::HTTP::"+verb.to_s.capitalize)
+      req = req_klass.new(uri.to_s, headers)
+      req.body = body if !body.nil? && !body.empty?
+
+      # Make the HTTP request
+      retries = max_retries
+      begin
+        raw_response = http.request(req)
+      rescue EOFError, Errno::EPIPE
+        # Something happened to our connection, lets try this again
+        if retries > 0
+          retries -= 1
+          open
+          retry
+        end 
+      end
+
+      # Let's make sure we close the connection if we don't intend to
+      # have persistent connections..
+      close if !keep_alive
+
+      # Build response
+      response = Response.new(raw_response, uri)
+
+      # TODO: what about redirects that go back and forth..
+      # should we introduce a max number of auto_redirects..?
+
+      # Handle URL Redirection
+      if auto_redirect && raw_response.kind_of?(Net::HTTPRedirection) && !response.header['location'].nil?
+        @uri = Addressable::URI.parse(response.header['location'])
+        request(verb, path, headers, body)
+      else
+        response
+      end
+    end
+
+  end
+end

data/lib/http-request/response.rb

@@ -0,0 +1,16 @@
+module HttpRequest
+  class Response
+    attr_reader :status, :header, :body, :raw, :request_uri
+
+    def initialize(net_http_response, request_uri)
+      @status       = net_http_response.code.to_i
+      @body         = net_http_response.body
+      @raw          = net_http_response
+      @request_uri  = request_uri
+
+      @header = net_http_response.header.to_hash.tap do |header|
+        header.each {|k,v| header[k] = v.first if v.is_a?(Array) && v.length == 1 }
+      end
+    end
+  end
+end

data/lib/http-request/version.rb

@@ -0,0 +1,3 @@
+module HttpRequest
+  VERSION = '1.1.0'
+end

data/lib/old.rb

@@ -0,0 +1,198 @@
+# +-------------------------------------------------------------------------------+
+# | HttpRequest                                                                   |
+# +-------------------------------------------------------------------------------+
+# | Author: Nulayer Inc. / www.nulayer.com                                        |
+# +-------------------------------------------------------------------------------+
+# | BSD LICENSE                                                                   |
+# | Copyright (c) 2007-2011, Nulayer Inc.                                         |
+# | All rights reserved.                                                          |
+# |                                                                               |
+# | Redistribution and use in source and binary forms, with or without            |
+# | modification, are permitted provided that the following conditions are met:   |
+# |     * Redistributions of source code must retain the above copyright          |
+# |       notice, this list of conditions and the following disclaimer.           |
+# |     * Redistributions in binary form must reproduce the above copyright       |
+# |       notice, this list of conditions and the following disclaimer in the     |
+# |       documentation and/or other materials provided with the distribution.    |
+# |     * Neither the name of the <organization> nor the                          |
+# |       names of its contributors may be used to endorse or promote products    |
+# |       derived from this software without specific prior written permission.   |
+# |                                                                               |
+# | THIS SOFTWARE IS PROVIDED BY NuLayer Inc. "AS IS" AND ANY                     |
+# | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED     |
+# | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE        |
+# | DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY            |
+# | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES    |
+# | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;  |
+# | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND   |
+# | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT    |
+# | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
+# | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.                  |
+# +-------------------------------------------------------------------------------+
+
+require 'net/http'
+
+class HttpRequest
+  VERSION = '1.0.5'
+
+  class HTTPError < StandardError; end
+  class UnexpectedError < StandardError; end
+
+  class UnreachableError < HTTPError; end
+  class ConnectionResetError < HTTPError; end
+  class TimeoutError < HTTPError; end
+  class RedirectError < HTTPError; end
+
+  attr_accessor :uri
+  attr_accessor :header
+  attr_accessor :body
+  attr_accessor :status
+
+  class << self
+    attr_accessor :user_agent, :timeout, :max_retries
+  end
+
+
+  def self.head(url, options={})
+    http = HttpRequest.new(url)
+    http.head(options)
+  end
+
+  def self.get(url, options={})
+    http = HttpRequest.new(url)
+    http.get(options)
+  end
+
+  def initialize(url)
+    self.uri = url
+  end
+
+  def head(options={})
+    request(:head, options)
+  end
+
+  def get(options={})
+    request(:get, options)
+  end
+
+  def request(type, options={})
+    user_agent     = options[:user_agent] || HttpRequest.user_agent || 'HttpRequest'
+    timeout        = (options[:timeout] || HttpRequest.timeout || 10).to_i
+    redirect_limit = options[:redirect_limit] || 5
+    output_file    = options[:output_file]
+    content_max    = options[:content_max]
+    max_retries     = HttpRequest.max_retries || 3
+
+    # Check if we're in a redirecting loop
+    if redirect_limit == 0
+      raise RedirectError, "HTTP redirect too deep: #{uri.to_s}"
+    end
+
+    # Prepare the request
+    req_path = uri.path
+    req_path += "?#{uri.query}" if !uri.query.nil?
+
+    req_klass = (type == :head ? Net::HTTP::Head : Net::HTTP::Get)
+    req = req_klass.new(req_path, { 'User-Agent' => user_agent })
+
+    # Basic authentication
+    if uri.user && uri.password
+      req.basic_auth uri.user, uri.password
+    end
+
+    # HTTP request block
+    @retries = max_retries
+    begin
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      http.open_timeout = timeout
+      http.read_timeout = timeout
+
+      http.start do |http|
+        http.request(req) do |response|
+          # Save the header
+          self.header = response.header
+          self.body   = nil
+
+          # Restrict downloading of content
+          download_content = true
+          if !content_max.nil? && header['content-length'].to_i > content_max
+            download_content = false
+          end
+
+          # Get the body
+          if download_content
+            if output_file
+              open(output_file, "w") do |f|
+                response.read_body { |data| f.write(data) }
+              end
+            else
+              self.body = response.read_body
+            end
+          end
+
+          # Handle URL Redirection
+          if response.kind_of?(Net::HTTPRedirection) && !header['location'].nil?
+            update_uri_location(header['location'])
+            request type, options.merge({ :redirect_limit => redirect_limit-1 })
+          end
+        end
+      end
+    rescue ::Errno::ENETUNREACH
+      raise UnreachableError, "network is unreachable: #{uri.to_s}"
+    rescue ::Errno::ECONNRESET, ::Errno::ECONNREFUSED, ::SocketError
+      raise ConnectionResetError, "connection reset by peer: #{uri.to_s}"
+    rescue ::Timeout::Error
+      raise TimeoutError, "connection timed out: #{uri.to_s}"
+    rescue ::Errno::EINTR => ex
+      # Transmission was interrupted, retry the request just once.
+      # This usually occurs because the process received a signal.
+      @retries -= 1
+      if @retries >= 0
+        retry
+      else
+        raise ex
+      end
+    rescue HTTPError => ex
+      # Catch exceptions that may be raised above -- not sure why we have to do this
+      raise ex
+    rescue => ex
+      raise UnexpectedError, "#{ex.inspect} -- #{uri.to_s}"
+    ensure
+      http.finish rescue nil
+      http = nil
+    end
+
+    return self
+  end
+
+  def header=(header)
+    return if header.nil?
+    @header     = header
+    self.status = header.code.to_i
+  end
+
+  def reset!
+    self.header = nil
+    self.body   = nil
+    self.status = nil
+  end
+
+  def uri=(url)
+    @uri = URI.parse(url)
+    @uri.path = '/' if @uri.path.empty?
+    reset!
+  end
+
+  def update_uri_location(location)
+    if header['location'].to_s[0].to_s.chr == '/'
+      url = "#{uri.scheme}://#{uri.host}"
+      url += ":#{uri.port}" if uri.port != 80 && uri.port != 443
+      url += location
+    else
+      url = location
+    end
+
+    self.uri = url
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http-request
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,56 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-08 00:00:00.000000000 Z
-dependencies: []
+date: 2012-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 description: Simple HTTP client built with Net/HTTP
 email:
 - peter.kieltyka@nulayer.com
@@ -18,9 +66,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- README
-- VERSION
+- README.md
+- lib/http-request/client.rb
+- lib/http-request/connection.rb
+- lib/http-request/response.rb
+- lib/http-request/version.rb
 - lib/http-request.rb
+- lib/old.rb
 homepage: http://nulayer.com
 licenses: []
 post_install_message: 
@@ -46,3 +98,4 @@ signing_key:
 specification_version: 3
 summary: Simple HTTP client built with Net/HTTP
 test_files: []
+has_rdoc: 

data/VERSION

@@ -1 +0,0 @@
-1.0.5