z-http-request 0.1.0

data/examples/socks5.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'ZMachine'
+require '../lib/z-http'
+
+ZMachine.run do
+  # Establish a SOCKS5 tunnel via SSH
+  # ssh -D 8000 some_remote_machine
+
+  connection_options = {:proxy => {:host => '127.0.0.1', :port => 8000, :type => :socks5}}
+  http = ZMachine::HttpRequest.new('http://igvita.com/', connection_options).get :redirects => 2
+
+  http.callback {
+    puts "#{http.response_header.status} - #{http.response.length} bytes\n"
+    puts http.response
+    ZMachine.stop
+  }
+
+  http.errback {
+    puts "Error: " + http.error
+    puts http.inspect
+    ZMachine.stop
+  }
+end

data/lib/z-http/client.rb

@@ -0,0 +1,318 @@
+require 'cookiejar'
+
+module ZMachine
+
+
+  class HttpClient
+    include Deferrable
+    include HttpEncoding
+    include HttpStatus
+
+    TRANSFER_ENCODING="TRANSFER_ENCODING"
+    CONTENT_ENCODING="CONTENT_ENCODING"
+    CONTENT_LENGTH="CONTENT_LENGTH"
+    CONTENT_TYPE="CONTENT_TYPE"
+    LAST_MODIFIED="LAST_MODIFIED"
+    KEEP_ALIVE="CONNECTION"
+    SET_COOKIE="SET_COOKIE"
+    LOCATION="LOCATION"
+    HOST="HOST"
+    ETAG="ETAG"
+
+    CRLF="\r\n"
+
+    attr_accessor :state, :response
+    attr_reader   :response_header, :error, :content_charset, :req, :cookies
+
+    def initialize(conn, options)
+      @conn = conn
+      @req  = options
+
+      @stream    = nil
+      @headers   = nil
+      @cookies   = []
+      @cookiejar = CookieJar.new
+
+      reset!
+    end
+
+    def reset!
+      @response_header = HttpResponseHeader.new
+      @state = :response_header
+
+      @response = ''
+      @error = nil
+      @content_decoder = nil
+      @content_charset = nil
+    end
+
+    def last_effective_url; @req.uri; end
+    def redirects; @req.followed; end
+    def peer; @conn.peer; end
+
+    def connection_completed
+      @state = :response_header
+
+      head, body = build_request, @req.body
+      @conn.middleware.each do |m|
+        head, body = m.request(self, head, body) if m.respond_to?(:request)
+      end
+
+      send_request(head, body)
+    end
+
+    def on_request_complete
+      begin
+        @content_decoder.finalize! if @content_decoder
+      rescue HttpDecoders::DecoderError
+        on_error "Content-decoder error"
+      end
+
+      unbind
+    end
+
+    def continue?
+      @response_header.status == 100 && (@req.method == 'POST' || @req.method == 'PUT')
+    end
+
+    def finished?
+      @state == :finished || (@state == :body && @response_header.content_length.nil?)
+    end
+
+    def redirect?
+      @response_header.redirection? && @req.follow_redirect?
+    end
+
+    def unbind(reason = nil)
+      if finished?
+        if redirect?
+
+          begin
+            @conn.middleware.each do |m|
+              m.response(self) if m.respond_to?(:response)
+            end
+
+            # one of the injected middlewares could have changed
+            # our redirect settings, check if we still want to
+            # follow the location header
+            if redirect?
+              @req.followed += 1
+
+              @cookies.clear
+              @cookies = @cookiejar.get(@response_header.location).map(&:to_s) if @req.pass_cookies
+              @req.set_uri(@response_header.location)
+
+              @conn.redirect(self)
+            else
+              succeed(self)
+            end
+
+          rescue Exception => e
+            on_error(e.message)
+          end
+        else
+          succeed(self)
+        end
+
+      else
+        on_error(reason || 'connection closed by server')
+      end
+    end
+
+    def on_error(msg = nil)
+      @error = msg
+      fail(self)
+    end
+    alias :close :on_error
+
+    def stream(&blk); @stream = blk; end
+    def headers(&blk); @headers = blk; end
+
+    def normalize_body(body)
+      body.is_a?(Hash) ? form_encode_body(body) : body
+    end
+
+    def build_request
+      head    = @req.headers ? munge_header_keys(@req.headers) : {}
+
+      if @conn.connopts.http_proxy?
+        proxy = @conn.connopts.proxy
+        head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
+      end
+
+      # Set the cookie header if provided
+      if cookie = head['cookie']
+        @cookies << encode_cookie(cookie) if cookie
+      end
+      head['cookie'] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
+
+      # Set connection close unless keepalive
+      if !@req.keepalive
+        head['connection'] = 'close'
+      end
+
+      # Set the Host header if it hasn't been specified already
+      head['host'] ||= encode_host
+
+      # Set the User-Agent if it hasn't been specified
+      if !head.key?('user-agent')
+        head['user-agent'] = "ZMachine HttpClient"
+      elsif head['user-agent'].nil?
+        head.delete('user-agent')
+      end
+
+      # Set the auth from the URI if given
+      head['Authorization'] = @req.uri.userinfo.split(/:/, 2) if @req.uri.userinfo
+
+      head
+    end
+
+    def send_request(head, body)
+      body    = normalize_body(body)
+      file    = @req.file
+      query   = @req.query
+
+      # Set the Content-Length if file is given
+      head['content-length'] = File.size(file) if file
+
+      # Set the Content-Length if body is given,
+      # or we're doing an empty post or put
+      if body
+        head['content-length'] = body.bytesize
+      elsif @req.method == 'POST' or @req.method == 'PUT'
+        # wont happen if body is set and we already set content-length above
+        head['content-length'] ||= 0
+      end
+
+      # Set content-type header if missing and body is a Ruby hash
+      if !head['content-type'] and @req.body.is_a? Hash
+        head['content-type'] = 'application/x-www-form-urlencoded'
+      end
+
+      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts.proxy)
+      request_header << encode_headers(head)
+      request_header << CRLF
+      @conn.send_data request_header
+
+      if body
+        @conn.send_data body
+      elsif @req.file
+        @conn.stream_file_data @req.file, :http_chunks => false
+      end
+    end
+
+    def on_body_data(data)
+      if @content_decoder
+        begin
+          @content_decoder << data
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      else
+        on_decoded_body_data(data)
+      end
+    end
+
+    def on_decoded_body_data(data)
+      data.force_encoding @content_charset if @content_charset
+      if @stream
+        @stream.call(data)
+      else
+        @response << data
+      end
+    end
+
+    def parse_response_header(header, version, status)
+      @response_header.raw = header
+      header.each do |key, val|
+        @response_header[key.upcase.gsub('-','_')] = val
+      end
+
+      @response_header.http_version = version.join('.')
+      @response_header.http_status  = status
+      @response_header.http_reason  = CODE[status] || 'unknown'
+
+      # invoke headers callback after full parse
+      # if one is specified by the user
+      @headers.call(@response_header) if @headers
+
+      unless @response_header.http_status and @response_header.http_reason
+        @state = :invalid
+        on_error "no HTTP response"
+        return
+      end
+
+      # add set-cookie's to cookie list
+      if @response_header.cookie && @req.pass_cookies
+        [@response_header.cookie].flatten.each {|cookie| @cookiejar.set(cookie, @req.uri)}
+      end
+
+      # correct location header - some servers will incorrectly give a relative URI
+      if @response_header.location
+        begin
+          location = Addressable::URI.parse(@response_header.location)
+          location.path = "/" if location.path.empty?
+
+          if location.relative?
+            location = @req.uri.join(location)
+          else
+            # if redirect is to an absolute url, check for correct URI structure
+            raise if location.host.nil?
+          end
+
+          @response_header[LOCATION] = location.to_s
+
+        rescue
+          on_error "Location header format error"
+          return
+        end
+      end
+
+      # Fire callbacks immediately after recieving header requests
+      # if the request method is HEAD. In case of a redirect, terminate
+      # current connection and reinitialize the process.
+      if @req.method == "HEAD"
+        @state = :finished
+        return
+      end
+
+      if @response_header.chunked_encoding?
+        @state = :chunk_header
+      elsif @response_header.content_length
+        @state = :body
+      else
+        @state = :body
+      end
+
+      if @req.decoding && decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
+        begin
+          @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      end
+
+      # handle malformed header - Content-Type repetitions.
+      content_type = [response_header[CONTENT_TYPE]].flatten.first
+
+      if String.method_defined?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(content_type)
+        @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
+      end
+    end
+
+    class CookieJar
+      def initialize
+        @jar = ::CookieJar::Jar.new
+      end
+
+      def set string, uri
+        @jar.set_cookie(uri, string) rescue nil # drop invalid cookies
+      end
+
+      def get uri
+        uri = URI.parse(uri) rescue nil
+        uri ? @jar.get_cookies(uri) : []
+      end
+    end # CookieJar
+  end
+end

data/lib/z-http/core_ext/bytesize.rb

@@ -0,0 +1,6 @@
+# bytesize was introduced in 1.8.7+
+if RUBY_VERSION <= "1.8.6"
+  class String
+    def bytesize; self.size; end
+  end
+end

data/lib/z-http/decoders.rb

@@ -0,0 +1,254 @@
+require 'zlib'
+require 'stringio'
+
+##
+# Provides a unified callback interface to decompression libraries.
+module ZMachine::HttpDecoders
+
+  class DecoderError < StandardError
+  end
+
+  class << self
+    def accepted_encodings
+      DECODERS.inject([]) { |r, d| r + d.encoding_names }
+    end
+
+    def decoder_for_encoding(encoding)
+      DECODERS.each { |d|
+        return d if d.encoding_names.include? encoding
+      }
+      nil
+    end
+  end
+
+  class Base
+    def self.encoding_names
+      name = to_s.split('::').last.downcase
+      [name]
+    end
+
+    ##
+    # chunk_callback:: [Block] To handle a decompressed chunk
+    def initialize(&chunk_callback)
+      @chunk_callback = chunk_callback
+    end
+
+    def <<(compressed)
+      return unless compressed && compressed.size > 0
+
+      decompressed = decompress(compressed)
+      receive_decompressed decompressed
+    end
+
+    def finalize!
+      decompressed = finalize
+      receive_decompressed decompressed
+    end
+
+    private
+
+    def receive_decompressed(decompressed)
+      if decompressed && decompressed.size > 0
+        @chunk_callback.call(decompressed)
+      end
+    end
+
+    protected
+
+    ##
+    # Must return a part of decompressed
+    def decompress(compressed)
+      nil
+    end
+
+    ##
+    # May return last part
+    def finalize
+      nil
+    end
+  end
+
+  class Deflate < Base
+    def decompress(compressed)
+      begin
+        @zstream ||= Zlib::Inflate.new(-Zlib::MAX_WBITS)
+        @zstream.inflate(compressed)
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+
+    def finalize
+      return nil unless @zstream
+
+      begin
+        r = @zstream.inflate(nil)
+        @zstream.close
+        r
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+  end
+
+  ##
+  # Partial implementation of RFC 1952 to extract the deflate stream from a gzip file
+  class GZipHeader
+    def initialize
+      @state = :begin
+      @data = ""
+      @pos = 0
+    end
+
+    def finished?
+      @state == :finish
+    end
+
+    def read(n, buffer)
+      if (@pos + n) <= @data.size
+        buffer << @data[@pos..(@pos + n - 1)]
+        @pos += n
+        return true
+      else
+        return false
+      end
+    end
+
+    def readbyte
+      if (@pos + 1) <= @data.size
+        @pos += 1
+        @data.getbyte(@pos - 1)
+      end
+    end
+
+    def eof?
+      @pos >= @data.size
+    end
+
+    def extract_stream(compressed)
+      @data << compressed
+      pos = @pos
+
+      while !eof? && !finished?
+        buffer = ""
+
+        case @state
+        when :begin
+          break if !read(10, buffer)
+
+          if buffer.getbyte(0) != 0x1f || buffer.getbyte(1) != 0x8b
+            raise DecoderError.new("magic header not found")
+          end
+
+          if buffer.getbyte(2) != 0x08
+            raise DecoderError.new("unknown compression method")
+          end
+
+          @flags = buffer.getbyte(3)
+          if (@flags & 0xe0).nonzero?
+            raise DecoderError.new("unknown header flags set")
+          end
+
+          # We don't care about these values, I'm leaving the code for reference
+          # @time = buffer[4..7].unpack("V")[0] # little-endian uint32
+          # @extra_flags = buffer.getbyte(8)
+          # @os = buffer.getbyte(9)
+
+          @state = :extra_length
+
+        when :extra_length
+          if (@flags & 0x04).nonzero?
+            break if !read(2, buffer)
+            @extra_length = buffer.unpack("v")[0] # little-endian uint16
+            @state = :extra
+          else
+            @state = :extra
+          end
+
+        when :extra
+          if (@flags & 0x04).nonzero?
+            break if read(@extra_length, buffer)
+            @state = :name
+          else
+            @state = :name
+          end
+
+        when :name
+          if (@flags & 0x08).nonzero?
+            while !(buffer = readbyte).nil?
+              if buffer == 0
+                @state = :comment
+                break
+              end
+            end
+          else
+            @state = :comment
+          end
+
+        when :comment
+          if (@flags & 0x10).nonzero?
+            while !(buffer = readbyte).nil?
+              if buffer == 0
+                @state = :hcrc
+                break
+              end
+            end
+          else
+            @state = :hcrc
+          end
+
+        when :hcrc
+          if (@flags & 0x02).nonzero?
+            break if !read(2, buffer)
+            @state = :finish
+          else
+            @state = :finish
+          end
+        end
+      end
+
+      if finished?
+        compressed[(@pos - pos)..-1]
+      else
+        ""
+      end
+    end
+  end
+
+  class GZip < Base
+    def self.encoding_names
+      %w(gzip compressed)
+    end
+
+    def decompress(compressed)
+      compressed.force_encoding('BINARY')
+      @header ||= GZipHeader.new
+      if !@header.finished?
+        compressed = @header.extract_stream(compressed)
+      end
+
+      @zstream ||= Zlib::Inflate.new(-Zlib::MAX_WBITS)
+      @zstream.inflate(compressed)
+    rescue Zlib::Error
+      raise DecoderError
+    end
+
+    def finalize
+      if @zstream
+        if !@zstream.finished?
+          r = @zstream.finish
+        end
+        @zstream.close
+        r
+      else
+        nil
+      end
+    rescue Zlib::Error
+      raise DecoderError
+    end
+
+  end
+
+  DECODERS = [Deflate, GZip]
+
+end

data/lib/z-http/http_client_options.rb

@@ -0,0 +1,51 @@
+class HttpClientOptions
+  attr_reader :uri, :method, :host, :port
+  attr_reader :headers, :file, :body, :query, :path
+  attr_reader :keepalive, :pass_cookies, :decoding
+
+  attr_accessor :followed, :redirects
+
+  def initialize(uri, options, method)
+    @keepalive = options[:keepalive] || false  # default to single request per connection
+    @redirects = options[:redirects] ||= 0     # default number of redirects to follow
+    @followed  = options[:followed]  ||= 0     # keep track of number of followed requests
+
+    @method   = method.to_s.upcase
+    @headers  = options[:head] || {}
+    @query    = options[:query]
+
+
+    @file     = options[:file]
+    @body     = options[:body]
+
+    @pass_cookies = options.fetch(:pass_cookies, true)  # pass cookies between redirects
+    @decoding     = options.fetch(:decoding, true)      # auto-decode compressed response
+
+    set_uri(uri, options[:path])
+  end
+
+  def follow_redirect?; @followed < @redirects; end
+  def ssl?; @uri.scheme == "https" || @uri.port == 443; end
+  def no_body?; @method == "HEAD"; end
+
+  def set_uri(uri, path = nil)
+    uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
+    uri.path = path if path
+    uri.path = '/' if uri.path.empty?
+
+    @uri = uri
+    @path = uri.path
+
+    # Make sure the ports are set as Addressable::URI doesn't
+    # set the port if it isn't there
+    if @uri.scheme == "https"
+      @uri.port ||= 443
+    else
+      @uri.port ||= 80
+    end
+
+    @host = @uri.host
+    @port = @uri.port
+
+  end
+end