igrigorik-em-http-request 0.1.3 → 0.1.5

data/.gitignore

@@ -0,0 +1,4 @@
+*.bundle
+*.o
+Makefile
+mkmf.log

data/README.rdoc

@@ -50,3 +50,20 @@ EventMachine based HTTP Request interface. Supports streaming response processin
     }
   }
 
+== POST example
+
+  EventMachine.run {
+    http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
+    http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
+      
+    # ...
+  }
+
+== Streaming body processing
+  EventMachine.run {
+    body = ''
+    on_body = lambda { |chunk| body += chunk }
+    http = EventMachine::HttpRequest.new('http://www.website.com/').get :on_response => on_body
+
+    # ...
+  }

data/em-http-request.gemspec

@@ -0,0 +1,42 @@
+spec = Gem::Specification.new do |s|
+  s.name = 'em-http-request'
+  s.version = '0.1.5'
+  s.date = '2009-03-20'
+  s.summary = 'EventMachine based HTTP Request interface'
+  s.description = s.summary
+  s.email = 'ilya@igvita.com'
+  s.homepage = "http://github.com/igrigorik/em-http-request"
+  s.has_rdoc = true
+  s.authors = ["Ilya Grigorik"]
+  s.add_dependency('eventmachine', '>= 0.12.2')
+  s.extensions = ["ext/buffer/extconf.rb" , "ext/http11_client/extconf.rb"]
+  s.rubyforge_project = "em-http-request" 
+
+  # ruby -rpp -e' pp `git ls-files`.split("\n") '
+  s.files = [
+ ".autotest",
+ ".gitignore",
+ "LICENSE",
+ "README.rdoc",
+ "Rakefile",
+ "em-http-request.gemspec",
+ "ext/buffer/em_buffer.c",
+ "ext/buffer/extconf.rb",
+ "ext/http11_client/ext_help.h",
+ "ext/http11_client/extconf.rb",
+ "ext/http11_client/http11_client.c",
+ "ext/http11_client/http11_parser.c",
+ "ext/http11_client/http11_parser.h",
+ "ext/http11_client/http11_parser.rl",
+ "lib/em-http.rb",
+ "lib/em-http/client.rb",
+ "lib/em-http/core_ext/hash.rb",
+ "lib/em-http/decoders.rb",
+ "lib/em-http/multi.rb",
+ "lib/em-http/request.rb",
+ "test/hash.rb",
+ "test/helper.rb",
+ "test/stallion.rb",
+ "test/test_multi.rb",
+ "test/test_request.rb"]
+end

data/lib/em-http.rb

@@ -6,12 +6,13 @@
 
 require 'rubygems'
 require 'eventmachine'
-require 'zlib'
 
 
 require File.dirname(__FILE__) + '/http11_client'
 require File.dirname(__FILE__) + '/em_buffer'
 
+require File.dirname(__FILE__) + '/em-http/core_ext/hash'
 require File.dirname(__FILE__) + '/em-http/client'
 require File.dirname(__FILE__) + '/em-http/multi'
-require File.dirname(__FILE__) + '/em-http/request'
+require File.dirname(__FILE__) + '/em-http/request'
+require File.dirname(__FILE__) + '/em-http/decoders'

data/lib/em-http/client.rb

@@ -1,3 +1,4 @@
+# -*- coding: undecided -*-
 # #--
 # Copyright (C)2008 Ilya Grigorik
 #
@@ -162,9 +163,10 @@ module EventMachine
 
       @state = :response_header
       @parser_nbytes = 0
-      @inflate = []
       @response = ''
+      @inflate = []
       @errors = ''
+      @content_decoder = nil
     end
 
     # start HTTP request once we establish connection to host
@@ -175,16 +177,11 @@ module EventMachine
 
     # request is done, invoke the callback
     def on_request_complete
-
-      if @response_header.compressed? and @inflate.include?(response_header[CONTENT_ENCODING])
-        case response_header[CONTENT_ENCODING]
-        when 'deflate' then 
-          @response = Zlib::Inflate.inflate(@response)
-        when 'gzip', 'compressed' then
-          @response = Zlib::GzipReader.new(StringIO.new(@response)).read
-        end
+      begin
+        @content_decoder.finalize! if @content_decoder
+      rescue HttpDecoders::DecoderError
+        on_error "Content-decoder error"
       end
-			
       unbind
     end
 
@@ -222,7 +219,13 @@ module EventMachine
     end
 
     def send_request_body
-      send_data @options[:body] if @options[:body]
+      return unless @options[:body]
+      if @options[:body].is_a? Hash
+        body = @options[:body].to_params
+      else
+        body = @options[:body]
+      end
+      send_data body
     end
 
     def receive_data(data)
@@ -232,11 +235,27 @@ module EventMachine
 
     # Called when part of the body has been read
     def on_body_data(data)
-      @response << data
+      if @content_decoder
+        begin
+          @content_decoder << data
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      else
+        on_decoded_body_data(data)
+      end
+    end
+
+    def on_decoded_body_data(data)
+      if (on_response = @options[:on_response])
+        on_response.call(data)
+      else
+        @response << data
+      end
     end
 
     def unbind
-      (@state == :finished) ? succeed : fail
+      (@state == :finished) ? succeed(self) : fail
       close_connection
     end
 
@@ -301,6 +320,15 @@ module EventMachine
         @bytes_remaining = @response_header.content_length
       end
 
+      if @inflate.include?(response_header[CONTENT_ENCODING]) &&
+          decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
+        begin
+          @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      end
+
       true
     end
 
@@ -401,6 +429,8 @@ module EventMachine
 
       false
     end
+
+  
   end
 
-end
+end

data/lib/em-http/core_ext/hash.rb

@@ -0,0 +1,53 @@
+class Hash
+  # Stolen partially from Merb : http://noobkit.com/show/ruby/gems/development/merb/hash/to_params.html
+  # Convert this hash to a query string:
+  #   
+  #   { :name => "Bob",
+  #     :address => {
+  #       :street => '111 Ruby Ave.',
+  #       :city => 'Ruby Central',
+  #       :phones => ['111-111-1111', '222-222-2222']
+  #     }
+  #   }.to_params
+  #   #=> "name=Bob&address[city]=Ruby Central&address[phones]=111-111-1111222-222-2222&address[street]=111 Ruby Ave."
+  # 
+  def to_params
+    params = ''
+    stack = []
+    
+    each do |k, v|
+      if v.is_a?(Hash)
+        stack << [k,v]
+      elsif v.is_a?(Array)
+        stack << [k,Hash.from_array(v)]
+      else
+        params << "#{k}=#{v}&"
+      end
+    end
+    
+    stack.each do |parent, hash|
+      hash.each do |k, v|
+        if v.is_a?(Hash)
+          stack << ["#{parent}[#{k}]", v]
+        else
+          params << "#{parent}[#{k}]=#{v}&"
+        end
+      end
+    end
+    
+    params.chop! # trailing &
+    params
+  end
+  
+  ##
+  # Builds a hash from an array with keys as array indices.
+  def self.from_array(array = [])
+    h = Hash.new
+    array.size.times do |t|
+      h[t] = array[t]
+    end
+    h
+  end
+
+end
+

data/lib/em-http/decoders.rb

@@ -0,0 +1,119 @@
+require 'zlib'
+
+##
+# Provides a unified callback interface to decompression libraries.
+module EventMachine::HttpDecoders
+
+  class DecoderError < StandardError
+  end
+
+  class << self
+    def accepted_encodings
+      DECODERS.inject([]) { |r,d| r + d.encoding_names }
+    end
+
+    def decoder_for_encoding(encoding)
+      DECODERS.each { |d|
+        return d if d.encoding_names.include? encoding
+      }
+      nil
+    end
+  end
+
+  class Base
+    def self.encoding_names
+      name = to_s.split('::').last.downcase
+      [name]
+    end
+
+    ##
+    # chunk_callback:: [Block] To handle a decompressed chunk
+    def initialize(&chunk_callback)
+      @chunk_callback = chunk_callback
+    end
+
+    def <<(compressed)
+      return unless compressed && compressed.size > 0
+
+      decompressed = decompress(compressed)
+      receive_decompressed decompressed
+    end
+
+    def finalize!
+      decompressed = finalize
+      receive_decompressed decompressed
+    end
+    
+    private
+
+    def receive_decompressed(decompressed)
+      if decompressed && decompressed.size > 0
+        @chunk_callback.call(decompressed)
+      end
+    end
+
+    protected
+
+    ##
+    # Must return a part of decompressed
+    def decompress(compressed)
+      nil
+    end
+
+    ##
+    # May return last part
+    def finalize
+      nil
+    end
+  end
+
+  class Deflate < Base
+    def decompress(compressed)
+      begin
+        @zstream ||= Zlib::Inflate.new(nil)
+        @zstream.inflate(compressed)
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+
+    def finalize
+      return nil unless @zstream
+
+      begin
+        r = @zstream.inflate(nil)
+        @zstream.close
+        r
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+  end
+
+  ##
+  # Oneshot decompressor, due to lack of a streaming Gzip reader
+  # implementation. We may steal code from Zliby to improve this.
+  #
+  # For now, do not put `gzip' or `compressed' in your accept-encoding
+  # header if you expect much data through the :on_response interface.
+  class GZip < Base
+    def self.encoding_names
+      %w(gzip compressed)
+    end
+
+    def decompress(compressed)
+      @buf ||= ''
+      @buf += compressed
+      nil
+    end
+
+    def finalize
+      Zlib::GzipReader.new(StringIO.new(@buf)).read
+    end
+  end
+
+  DECODERS = [Deflate, GZip]
+
+end
+
+    

data/lib/em-http/request.rb

@@ -27,7 +27,7 @@ module EventMachine
 
     def initialize(host, headers = {})
       @headers = headers
-      @uri = URI::parse(host)
+      @uri = URI::parse(host) unless host.kind_of? URI
     end
 
     # Send an HTTP request and consume the response. Supported options:
@@ -41,6 +41,10 @@ module EventMachine
     #   body: String
     #     Specify the request body (you must encode it for now)
     #
+    #   on_response: Proc
+    #     Called for each response body chunk (you may assume HTTP 200
+    #     OK then)
+    #
 
     def get  options = {};    send_request(:get,  options);    end
     def post options = {};    send_request(:post, options);    end
@@ -66,4 +70,4 @@ module EventMachine
       end
     end
   end
-end
+end

data/test/hash.rb

@@ -0,0 +1,17 @@
+require 'test/helper'
+
+describe Hash do
+  describe ".to_params" do
+    it "should transform a basic hash into HTTP POST Params" do
+      {:a => "alpha", :b => "beta"}.to_params.should == "a=alpha&b=beta"
+    end
+    
+    it "should transform a more complex hash into HTTP POST Params" do
+      {:a => "a", :b => ["c", "d", "e"]}.to_params.should == "a=a&b[0]=c&b[1]=d&b[2]=e"
+    end
+    
+    it "should transform a very complex hash into HTTP POST Params" do
+      {:a => "a", :b => [{:c => "c", :d => "d"}, {:e => "e", :f => "f"}]}.to_params.should == "a=a&b[0][d]=d&b[0][c]=c&b[1][e]=e&b[1][f]=f"
+    end
+  end
+end

data/test/stallion.rb

@@ -81,7 +81,7 @@ Stallion.saddle :spec do |stable|
       stable.response.write stable.request.query_string
 
     elsif stable.request.post?
-      stable.response.write 'test'
+      stable.response.write stable.request.body.read
 
     elsif stable.request.path_info == '/timeout'
       sleep(10)

data/test/test_request.rb

@@ -19,7 +19,7 @@ describe EventMachine::HttpRequest do
     }
   end
 
- it "should fail GET on invalid host" do
+  it "should fail GET on invalid host" do
     EventMachine.run {
       http = EventMachine::HttpRequest.new('http://google1.com/').get
       http.callback { failed }
@@ -112,7 +112,21 @@ describe EventMachine::HttpRequest do
       http.errback { failed }
       http.callback {
         http.response_header.status.should == 200
-        http.response.should match(/test/)
+        http.response.should match(/data/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should perform successfull POST with Ruby Hash/Array as params" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').post :body => {"key1" => 1, "key2" => [2,3]}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        
+        http.response.should match(/key1=1&key2\[0\]=2&key2\[1\]=3/)
         EventMachine.stop
       }
     }
@@ -214,4 +228,38 @@ describe EventMachine::HttpRequest do
       http.callback { failed }
     }
   end
-end
+
+  it "should optionally pass the response body progressively" do
+    EventMachine.run {
+      body = ''
+      on_body = lambda { |chunk| body += chunk }
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :on_response => on_body
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == ''
+        body.should match(/Hello/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should optionally pass the deflate-encoded response body progressively" do
+    EventMachine.run {
+      body = ''
+      on_body = lambda { |chunk| body += chunk }
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/deflate').get :head => {"accept-encoding" => "deflate, compressed"},
+      :on_response => on_body
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "deflate"
+        http.response.should == ''
+        body.should == "compressed"
+        EventMachine.stop
+      }
+    }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: igrigorik-em-http-request
 version: !ruby/object:Gem::Version 
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -33,9 +33,11 @@ extra_rdoc_files: []
 
 files: 
 - .autotest
+- .gitignore
 - LICENSE
 - README.rdoc
 - Rakefile
+- em-http-request.gemspec
 - ext/buffer/em_buffer.c
 - ext/buffer/extconf.rb
 - ext/http11_client/ext_help.h
@@ -46,8 +48,11 @@ files:
 - ext/http11_client/http11_parser.rl
 - lib/em-http.rb
 - lib/em-http/client.rb
+- lib/em-http/core_ext/hash.rb
+- lib/em-http/decoders.rb
 - lib/em-http/multi.rb
 - lib/em-http/request.rb
+- test/hash.rb
 - test/helper.rb
 - test/stallion.rb
 - test/test_multi.rb