astro-em-http-request 0.1.3.20090419

data/lib/em-http/decoders.rb

@@ -0,0 +1,119 @@
+require 'zlib'
+
+##
+# Provides a unified callback interface to decompression libraries.
+module EventMachine::HttpDecoders
+
+  class DecoderError < StandardError
+  end
+
+  class << self
+    def accepted_encodings
+      DECODERS.inject([]) { |r,d| r + d.encoding_names }
+    end
+
+    def decoder_for_encoding(encoding)
+      DECODERS.each { |d|
+        return d if d.encoding_names.include? encoding
+      }
+      nil
+    end
+  end
+
+  class Base
+    def self.encoding_names
+      name = to_s.split('::').last.downcase
+      [name]
+    end
+
+    ##
+    # chunk_callback:: [Block] To handle a decompressed chunk
+    def initialize(&chunk_callback)
+      @chunk_callback = chunk_callback
+    end
+
+    def <<(compressed)
+      return unless compressed && compressed.size > 0
+
+      decompressed = decompress(compressed)
+      receive_decompressed decompressed
+    end
+
+    def finalize!
+      decompressed = finalize
+      receive_decompressed decompressed
+    end
+    
+    private
+
+    def receive_decompressed(decompressed)
+      if decompressed && decompressed.size > 0
+        @chunk_callback.call(decompressed)
+      end
+    end
+
+    protected
+
+    ##
+    # Must return a part of decompressed
+    def decompress(compressed)
+      nil
+    end
+
+    ##
+    # May return last part
+    def finalize
+      nil
+    end
+  end
+
+  class Deflate < Base
+    def decompress(compressed)
+      begin
+        @zstream ||= Zlib::Inflate.new(nil)
+        @zstream.inflate(compressed)
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+
+    def finalize
+      return nil unless @zstream
+
+      begin
+        r = @zstream.inflate(nil)
+        @zstream.close
+        r
+      rescue Zlib::Error
+        raise DecoderError
+      end
+    end
+  end
+
+  ##
+  # Oneshot decompressor, due to lack of a streaming Gzip reader
+  # implementation. We may steal code from Zliby to improve this.
+  #
+  # For now, do not put `gzip' or `compressed' in your accept-encoding
+  # header if you expect much data through the :on_response interface.
+  class GZip < Base
+    def self.encoding_names
+      %w(gzip compressed)
+    end
+
+    def decompress(compressed)
+      @buf ||= ''
+      @buf += compressed
+      nil
+    end
+
+    def finalize
+      Zlib::GzipReader.new(StringIO.new(@buf)).read
+    end
+  end
+
+  DECODERS = [Deflate, GZip]
+
+end
+
+    

data/lib/em-http/multi.rb

@@ -0,0 +1,51 @@
+module EventMachine
+
+  # EventMachine based Multi request client, based on a streaming HTTPRequest class, 
+  # which allows you to open multiple parallel connections and return only when all
+  # of them finish. (i.e. ideal for parallelizing workloads)
+  # 
+  # == Example
+  # 
+  #  EventMachine.run {
+  #   
+  #   multi = EventMachine::MultiRequest.new
+  #    
+  #   # add multiple requests to the multi-handler
+  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+  #    
+  #    multi.callback  {
+  #       p multi.responses[:succeeded]
+  #       p multi.responses[:failed]
+  #     
+  #       EventMachine.stop
+  #    }
+  #  }
+  # 
+  
+  class MultiRequest
+    include EventMachine::Deferrable
+
+    attr_reader :requests, :responses
+    
+    def initialize
+      @requests = []
+      @responses = {:succeeded => [], :failed => []}
+    end
+    
+    def add(conn)
+      conn.callback { @responses[:succeeded].push(conn); check_progress }
+      conn.errback  { @responses[:failed].push(conn); check_progress }
+      
+      @requests.push(conn)
+    end
+    
+    protected
+   
+    # invoke callback if all requests have completed
+    def check_progress
+      succeed if (@responses[:succeeded].size + @responses[:failed].size) == @requests.size
+    end
+    
+  end
+end

data/lib/em-http/request.rb

@@ -0,0 +1,77 @@
+require 'uri'
+require 'base64'
+
+module EventMachine
+
+  # EventMachine based HTTP request class with support for streaming consumption
+  # of the response. Response is parsed with a Ragel-generated whitelist parser
+  # which supports chunked HTTP encoding.
+  #
+  # == Example
+  #
+  #
+  #  EventMachine.run {
+  #    http = EventMachine::HttpRequest.new('http://127.0.0.1/').get :query => {'keyname' => 'value'}
+  #
+  #    http.callback {
+  #     p http.response_header.status
+  #     p http.response_header
+  #     p http.response
+  #
+  #	EventMachine.stop
+  #    }
+  #  }
+  #
+
+  class HttpRequest
+
+    def initialize(host, headers = {})
+      @headers = headers
+      @uri = URI::parse(host) unless host.kind_of? URI
+    end
+
+    # Send an HTTP request and consume the response. Supported options:
+    #
+    #   head: {Key: Value}
+    #     Specify an HTTP header, e.g. {'Connection': 'close'}
+    #
+    #   query: {Key: Value}
+    #     Specify query string parameters (auto-escaped)
+    #
+    #   body: String
+    #     Specify the request body (you must encode it for now)
+    #
+    #   on_response: Proc
+    #     Called for each response body chunk (you may assume HTTP 200
+    #     OK then)
+    #
+    #   host: String
+    #     Manually specify TCP connect host address, independent of
+    #     Host: header
+
+    def get  options = {};    send_request(:get,  options);    end
+    def post options = {};    send_request(:post, options);    end
+
+    protected
+
+    def send_request(method, options)
+      raise ArgumentError, "invalid request path" unless /^\// === @uri.path
+
+      method = method.to_s.upcase
+      begin
+       host = options[:host] || @uri.host
+       EventMachine.connect(host, @uri.port, EventMachine::HttpClient) { |c|
+          c.uri = @uri
+          c.method = method
+          c.options = options
+          c.comm_inactivity_timeout = options[:timeout] || 5
+        }
+      rescue RuntimeError => e 
+        raise e unless e.message == "no connection"
+        conn = EventMachine::HttpClient.new("")
+        conn.on_error("no connection")
+        conn
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'spec'
+require 'pp'
+
+require 'lib/em-http'

data/test/stallion.rb

@@ -0,0 +1,127 @@
+# #--
+# Includes portion originally Copyright (C)2008 Michael Fellinger
+# license See file LICENSE for details
+# #--
+
+require 'rack'
+
+module Stallion
+  class Mount
+    def initialize(name, *methods, &block)
+      @name, @methods, @block = name, methods, block
+    end
+
+    def ride
+      @block.call
+    end
+
+    def match?(request)
+      method = request['REQUEST_METHOD']
+      right_method = @methods.empty? or @methods.include?(method)
+    end
+  end
+
+  class Stable
+    attr_reader :request, :response
+
+    def initialize
+      @boxes = {}
+    end
+
+    def in(path, *methods, &block)
+      mount = Mount.new(path, *methods, &block)
+      @boxes[[path, methods]] = mount
+      mount
+    end
+
+    def call(request, response)
+      @request, @response = request, response
+      @boxes.each do |(path, methods), mount|
+        if mount.match?(request)
+          mount.ride
+        end
+      end
+    end
+  end
+
+  STABLES = {}
+
+  def self.saddle(name = nil)
+    STABLES[name] = stable = Stable.new
+    yield stable
+  end
+
+  def self.run(options = {})
+    options = {:Host => "127.0.0.1", :Port => 8080}.merge(options)
+    Rack::Handler::Mongrel.run(Rack::Lint.new(self), options)
+  end
+
+  def self.call(env)
+    request = Rack::Request.new(env)
+    response = Rack::Response.new
+
+    STABLES.each do |name, stable|
+      stable.call(request, response)
+    end
+
+    response.finish
+  end
+end
+
+Stallion.saddle :spec do |stable|
+  stable.in '/' do
+
+    if stable.request.path_info == '/fail'
+      stable.response.status = 404
+
+    elsif stable.request.query_string == 'q=test'
+      stable.response.write 'test'
+
+    elsif stable.request.path_info == '/echo_query'
+      stable.response.write stable.request.query_string
+
+    elsif stable.request.post?
+      stable.response.write 'test'
+
+    elsif stable.request.path_info == '/timeout'
+      sleep(10)
+      stable.response.write 'timeout'
+
+    elsif stable.request.path_info == '/gzip'
+      io = StringIO.new
+      gzip = Zlib::GzipWriter.new(io)
+      gzip << "compressed"
+      gzip.close
+
+      stable.response.write io.string
+      stable.response["Content-Encoding"] = "gzip"
+
+    elsif stable.request.path_info == '/deflate'
+      stable.response.write Zlib::Deflate.deflate("compressed")
+      stable.response["Content-Encoding"] = "deflate"
+
+    elsif stable.request.env["HTTP_IF_NONE_MATCH"]
+      stable.response.status = 304
+
+    elsif stable.request.env["HTTP_AUTHORIZATION"]
+      auth = "Basic %s" % Base64.encode64(['user', 'pass'].join(':')).chomp
+
+      if auth == stable.request.env["HTTP_AUTHORIZATION"]
+        stable.response.status = 200
+        stable.response.write 'success'
+      else
+        stable.response.status = 401
+      end
+
+    elsif
+      stable.response.write  'Hello, World!'
+    end
+
+  end
+end
+
+Thread.new do
+  Stallion.run :Host => '127.0.0.1', :Port => 8080
+end
+
+sleep(2)

data/test/test_multi.rb

@@ -0,0 +1,34 @@
+require 'test/helper'
+require 'test/stallion'
+
+describe EventMachine::MultiRequest do
+
+  def failed
+    EventMachine.stop
+    fail
+  end
+
+  it "should submit multiple requests in parallel and return once all of them are complete" do
+    EventMachine.run {
+      
+      # create an instance of multi-request handler, and the requests themselves
+      multi = EventMachine::MultiRequest.new
+      
+      # add multiple requests to the multi-handler
+      multi.add(EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get(:query => {:q => 'test'}))
+      multi.add(EventMachine::HttpRequest.new('http://169.169.169.169/').get)
+      
+      multi.callback  {
+        # verify successfull request
+        multi.responses[:succeeded].size.should == 1
+        multi.responses[:succeeded].first.response.should match(/test/)
+        
+        # verify invalid requests
+        multi.responses[:failed].size.should == 1
+        multi.responses[:failed].first.response_header.status.should == 0
+        
+        EventMachine.stop
+      }
+    } 
+  end
+end

data/test/test_request.rb

@@ -0,0 +1,264 @@
+require 'test/helper'
+require 'test/stallion'
+
+describe EventMachine::HttpRequest do
+
+  def failed
+    EventMachine.stop
+    fail
+  end
+
+  it "should fail GET on DNS timeout" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.1.1.1/').get
+      http.callback { failed }
+      http.errback {
+        http.response_header.status.should == 0
+        EventMachine.stop
+      }
+    }
+  end
+
+ it "should fail GET on invalid host" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://google1.com/').get
+      http.callback { failed }
+      http.errback {
+        http.response_header.status.should == 0
+        http.errors.should match(/no connection/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should fail GET on missing path" do
+    EventMachine.run {
+      lambda {
+        EventMachine::HttpRequest.new('http://www.google.com').get
+      }.should raise_error(ArgumentError)
+
+      EventMachine.stop
+    }
+  end
+
+  it "should perform successfull GET" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/Hello/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should return 404 on invalid path" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/fail').get
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 404
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should build query parameters from Hash" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :query => {:q => 'test'}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/test/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should pass query parameters string" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :query => "q=test"
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/test/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should encode an array of query parameters" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/echo_query').get :query => {:hash => ['value1', 'value2']}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/hash\[\]=value1&hash\[\]=value2/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should perform successfull POST" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').post :body => "data"
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/test/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should perform successfull GET with custom header" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :head => {'if-none-match' => 'evar!'}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 304
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should perform a streaming GET" do
+    EventMachine.run {
+
+      # digg.com uses chunked encoding
+      http = EventMachine::HttpRequest.new('http://digg.com/').get
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should perform basic auth" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :head => {'authorization' => ['user', 'pass']}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should work with keep-alive servers" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://mexicodiario.com/touch.public.json.php').get
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should detect deflate encoding" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/deflate').get :head => {"accept-encoding" => "deflate"}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "deflate"
+        http.response.should == "compressed"
+
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should detect gzip encoding" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/gzip').get :head => {"accept-encoding" => "gzip, compressed"}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "gzip"
+        http.response.should == "compressed"
+
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should timeout after 10 seconds" do
+    EventMachine.run {
+      t = Time.now.to_i
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/timeout').get :timeout => 2
+
+      http.errback {
+        (Time.now.to_i - t).should == 2
+        EventMachine.stop
+      }
+      http.callback { failed }
+    }
+  end
+
+  it "should optionally pass the response body progressively" do
+    EventMachine.run {
+      body = ''
+      on_body = lambda { |chunk| body += chunk }
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :on_response => on_body
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == ''
+        body.should match(/Hello/)
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should optionally pass the deflate-encoded response body progressively" do
+    EventMachine.run {
+      body = ''
+      on_body = lambda { |chunk| body += chunk }
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/deflate').get :head => {"accept-encoding" => "deflate, compressed"},
+                                                                             :on_response => on_body
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "deflate"
+        http.response.should == ''
+        body.should == "compressed"
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should respect manually-passed host address" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.1.1.1:8080/').get :host => '127.0.0.1'
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/Hello/)
+        EventMachine.stop
+      }
+    }
+  end
+end