em-http-request 0.2.9 → 0.3.0

data/.gitignore

@@ -1,4 +1,5 @@
 *.bundle
 *.o
 Makefile
-mkmf.log
+mkmf.log
+Gemfile.lock

data/Changelog.md

@@ -0,0 +1,54 @@
+# Changelog
+
+## 0.3.0 / 2011-01-15
+
+- IMPORTANT: default to non-persistent connections (timeout => 0 now requires :keepalive => true)
+- see: https://github.com/igrigorik/em-http-request/commit/1ca5b608e876c18fa6cfa318d0685dcf5b974e09
+
+- added escape_utils dependency to fix slow encode on long string escapes
+
+- bugfix: proxy authorization headers
+- bugfix: default to Encoding.default_external on invalid encoding in response
+- bugfix: do not normalize URI's internally
+- bugfix: more robust Encoding detection
+
+
+## 0.2.15 / 2010-11-18
+
+- bugfix: follow redirects on missing content-length
+- bugfix: fixed undefined warnings when running in strict mode
+
+## 0.2.14 / 2010-10-06
+
+- bugfix: form-encode keys/values of ruby objects passed in as body
+
+## 0.2.13 / 2010-09-25
+
+- added SOCKS5 proxy support
+- bugfix: follow redirects on HEAD requests
+
+## 0.2.12 / 2010-09-12
+
+- added headers callback (http.headers {|h| p h})
+- added .close method on client obj to terminate session (accepts message)
+
+- bugfix: report 0 for response status on 1.9 on timeouts
+- bugfix: handle bad Location host redirects
+- bugfix: reset host override on connect
+
+## 0.2.11 / 2010-08-16
+
+- all URIs are now normalized prior to dispatch (and on redirect)
+- default to direct proxy (instead of CONNECT handshake) - better performance
+  - specify :proxy => {:tunnel => true} if you need to force CONNECT route
+- MultiRequest accepts block syntax for dispatching parallel requests (see specs)
+- MockHttpRequest accepts block syntax (see Mock wiki page)
+
+
+- bugfix: nullbyte frame for websockets
+- bugfix: set @uri on DNS resolve failure
+- bugfix: handle bad hosts in absolute redirects
+- bugfix: reset seen content on redirects (doh!)
+- bugfix: invalid multibyte escape in websocket regex (1.9.2+)
+- bugfix: report etag and last_modified headers correctly
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,175 @@
+EM-HTTP-Request
+===============
+
+Asynchronous HTTP client for Ruby, based on EventMachine runtime.
+
+- Ragel HTTP parser for speed & performance
+- Simple interface for single & parallel requests via deferred callbacks
+- Automatic gzip & deflate decoding
+- Basic-Auth & OAuth support
+- Custom timeout support
+- Stream response processing
+- Proxy support (with SSL Tunneling): CONNECT, direct & SOCKS5
+- Auto-follow 3xx redirects with custom max depth
+- Bi-directional communication with web-socket services
+- [Native mocking support](http://wiki.github.com/igrigorik/em-http-request/mocking-httprequest) and through [Webmock](http://github.com/bblimke/webmock)
+
+Getting started
+---------------
+
+      gem install em-http-request
+      irb:0> require 'em-http'
+
+Or checkout [screencast / demo](http://everburning.com/news/eventmachine-screencast-em-http-request/) of using EM-HTTP-Request.
+
+Libraries & Applications using em-http
+--------------------------------------
+
+- [chirpstream](http://github.com/joshbuddy/chirpstream) - EM client for Twitters Chirpstream API
+- [RDaneel](http://github.com/hasmanydevelopers/RDaneel) - Ruby crawler which respects robots.txt
+- [rsolr-async](http://github.com/mwmitchell/rsolr-async) - An asynchronus connection adapter for RSolr
+- [PubSubHubbub](http://github.com/igrigorik/PubSubHubbub) - Asynchronous PubSubHubbub ruby client
+- [Firering](http://github.com/EmmanuelOga/firering) - Eventmachine powered Campfire API
+- and many others.. drop me a link if you want yours included!
+
+Simple client example
+---------------------
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://127.0.0.1/').get :query => {'keyname' => 'value'}, :timeout => 10
+
+        http.callback {
+          p http.response_header.status
+          p http.response_header
+          p http.response
+
+          EventMachine.stop
+        }
+      }
+
+Multi-request example
+---------------------
+
+Fire and wait for multiple requests to complete via the MultiRequest interface.
+
+      EventMachine.run {
+        multi = EventMachine::MultiRequest.new
+
+        # add multiple requests to the multi-handler
+        multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+        multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+
+        multi.callback  {
+          p multi.responses[:succeeded]
+          p multi.responses[:failed]
+
+          EventMachine.stop
+        }
+      }
+
+Basic-Auth example
+------------------
+
+Full basic author support. For OAuth, check examples/oauth-tweet.rb file.
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.website.com/').get :head => {'authorization' => ['user', 'pass']}
+
+        http.errback { failed }
+        http.callback {
+          p http.response_header
+          EventMachine.stop
+        }
+      }
+
+
+POSTing data example
+--------------------
+
+For multi-part uploads, please see [this gist](https://gist.github.com/778639).
+
+      EventMachine.run {
+        http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
+        http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
+
+        # ...
+      }
+
+Streaming body processing
+-------------------------
+
+Allows you to consume an HTTP stream of content in real-time. Each time a new piece of content is pushed
+to the client, it is passed to the stream callback for you to operate on.
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.website.com/').get
+        http.stream { |chunk| print chunk }
+      }
+
+Streaming files from disk
+-------------------------
+Allows you to efficiently stream a (large) file from disk via EventMachine's FileStream interface.
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.website.com/').post :file => 'largefile.txt'
+        http.callback { |chunk| puts "Upload finished!" }
+      }
+
+Proxy example
+-------------
+Full transparent proxy support with support for SSL tunneling.
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
+          :host => 'www.myproxy.com',
+          :port => 8080,
+          :authorization => ['username', 'password'] # authorization is optional
+      }
+
+SOCKS5 Proxy example
+-------------
+Tunnel your requests via connect via SOCKS5 proxies (ssh -D port somehost).
+
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
+        :host => 'www.myproxy.com',
+        :port => 8080,
+        :type => :socks
+    }
+
+Auto-follow 3xx redirects
+-------------------------
+
+Specify the max depth of redirects to follow, default is 0.
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.google.com/').get :redirects => 1
+        http.callback { p http.last_effective_url }
+      }
+
+WebSocket example
+-----------------
+
+[Bi-directional communication with WebSockets](http://www.igvita.com/2009/12/22/ruby-websockets-tcp-for-the-browser/): simply pass in a ws:// resource and the client will negotiate the connection upgrade for you. On successful handshake the callback is invoked, and any incoming messages will be passed to the stream callback. The client can also send data to the server at will by calling the "send" method!
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new("ws://yourservice.com/websocket").get :timeout => 0
+
+        http.errback { puts "oops" }
+        http.callback {
+          puts "WebSocket connected!"
+          http.send("Hello client")
+        }
+
+        http.stream { |msg|
+          puts "Recieved: #{msg}"
+          http.send "Pong: #{msg}"
+        }
+
+        http.disconnect { puts "oops, dropped connection?" }
+      }
+
+License
+-------
+
+(MIT License) - Copyright (c) 2011 Ilya Grigorik

data/Rakefile

@@ -1,7 +1,13 @@
+require 'bundler'
+
+Bundler.setup
+Bundler.require :default, :development
+
 require 'rake'
 require 'rake/clean'
-require 'rake/rdoctask'
 require 'rake/gempackagetask'
+require 'rspec/core/rake_task'
+
 require 'fileutils'
 include FileUtils
 
@@ -15,15 +21,6 @@ end
 # Default Rake task is compile
 task :default => :compile
 
-# RDoc
-Rake::RDocTask.new(:rdoc) do |task|
-  task.rdoc_dir = 'doc'
-  task.title    = 'EventMachine::HttpRequest'
-  task.options = %w(--title HttpRequest --main README --line-numbers)
-  task.rdoc_files.include(['lib/**/*.rb'])
-  task.rdoc_files.include(['README', 'LICENSE'])
-end
-
 # Rebuild parser Ragel
 task :ragel do
   Dir.chdir "ext/http11_client" do
@@ -34,9 +31,8 @@ task :ragel do
   end
 end
 
-task :spec do
-  sh 'spec spec/*_spec.rb'
-end
+desc "Run all RSpec tests"
+RSpec::Core::RakeTask.new(:spec)
 
 def make(makedir)
   Dir.chdir(makedir) { sh MAKE }
@@ -48,7 +44,7 @@ end
 
 def setup_extension(dir, extension)
   ext = "ext/#{dir}"
-  ext_so = "#{ext}/#{extension}.#{Config::CONFIG['DLEXT']}"
+  ext_so = "#{ext}/#{extension}.#{Config::MAKEFILE_CONFIG['DLEXT']}"
   ext_files = FileList[
     "#{ext}/*.c",
     "#{ext}/*.h",
@@ -81,26 +77,4 @@ setup_extension("http11_client", "http11_client")
 task :compile => [:em_buffer, :http11_client]
 
 CLEAN.include ['build/*', '**/*.o', '**/*.so', '**/*.a', '**/*.log', 'pkg']
-CLEAN.include ['ext/buffer/Makefile', 'lib/em_buffer.*', 'lib/http11_client.*']
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "em-http-request"
-    gemspec.summary = "EventMachine based, async HTTP Request interface"
-    gemspec.description = gemspec.summary
-    gemspec.email = "ilya@igvita.com"
-    gemspec.homepage = "http://github.com/igrigorik/em-http-request"
-    gemspec.authors = ["Ilya Grigorik"]
-    gemspec.required_ruby_version = ">= 1.8.6"
-    gemspec.extensions = ["ext/buffer/extconf.rb" , "ext/http11_client/extconf.rb"]
-    gemspec.add_dependency('eventmachine', '>= 0.12.9')
-    gemspec.add_dependency('addressable', '>= 2.0.0')
-    gemspec.rubyforge_project = "em-http-request"
-    gemspec.files = FileList[`git ls-files`.split]
-  end
-
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
-end
+CLEAN.include ['ext/buffer/Makefile', 'lib/em_buffer.*', 'lib/http11_client.*']

data/em-http-request.gemspec

@@ -1,97 +1,32 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-http/version"
 
 Gem::Specification.new do |s|
-  s.name = %q{em-http-request}
-  s.version = "0.2.9"
+  s.name        = "em-http-request"
+  s.version     = EventMachine::HttpRequest::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/em-http-request"
+  s.summary     = "EventMachine based, async HTTP Request client"
+  s.description = s.summary
+  s.rubyforge_project = "em-http-request"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Ilya Grigorik"]
-  s.date = %q{2010-07-07}
-  s.description = %q{EventMachine based, async HTTP Request interface}
-  s.email = %q{ilya@igvita.com}
-  s.extensions = ["ext/buffer/extconf.rb", "ext/http11_client/extconf.rb"]
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
-  ]
-  s.files = [
-    ".gitignore",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "em-http-request.gemspec",
-     "examples/fetch.rb",
-     "examples/fibered-http.rb",
-     "examples/oauth-tweet.rb",
-     "examples/websocket-handler.rb",
-     "examples/websocket-server.rb",
-     "ext/buffer/em_buffer.c",
-     "ext/buffer/extconf.rb",
-     "ext/http11_client/ext_help.h",
-     "ext/http11_client/extconf.rb",
-     "ext/http11_client/http11_client.c",
-     "ext/http11_client/http11_parser.c",
-     "ext/http11_client/http11_parser.h",
-     "ext/http11_client/http11_parser.rl",
-     "lib/em-http-request.rb",
-     "lib/em-http.rb",
-     "lib/em-http/client.rb",
-     "lib/em-http/core_ext/bytesize.rb",
-     "lib/em-http/core_ext/hash.rb",
-     "lib/em-http/decoders.rb",
-     "lib/em-http/http_options.rb",
-     "lib/em-http/mock.rb",
-     "lib/em-http/multi.rb",
-     "lib/em-http/request.rb",
-     "spec/fixtures/google.ca",
-     "spec/hash_spec.rb",
-     "spec/helper.rb",
-     "spec/mock_spec.rb",
-     "spec/multi_spec.rb",
-     "spec/request_spec.rb",
-     "spec/stallion.rb",
-     "spec/stub_server.rb"
-  ]
-  s.homepage = %q{http://github.com/igrigorik/em-http-request}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.required_ruby_version = Gem::Requirement.new(">= 1.8.6")
-  s.rubyforge_project = %q{em-http-request}
-  s.rubygems_version = %q{1.3.6}
-  s.summary = %q{EventMachine based, async HTTP Request interface}
-  s.test_files = [
-    "spec/hash_spec.rb",
-     "spec/helper.rb",
-     "spec/mock_spec.rb",
-     "spec/multi_spec.rb",
-     "spec/request_spec.rb",
-     "spec/stallion.rb",
-     "spec/stub_server.rb",
-     "examples/fetch.rb",
-     "examples/fibered-http.rb",
-     "examples/oauth-tweet.rb",
-     "examples/websocket-handler.rb",
-     "examples/websocket-server.rb"
-  ]
+  s.add_dependency "eventmachine", ">= 0.12.9"
+  s.add_dependency "addressable", ">= 2.0.0"
+  s.add_dependency "escape_utils"
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "em-websocket"
+  s.add_development_dependency "rack"
+  s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.9"])
-      s.add_runtime_dependency(%q<addressable>, [">= 2.0.0"])
-    else
-      s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
-      s.add_dependency(%q<addressable>, [">= 2.0.0"])
-    end
-  else
-    s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
-    s.add_dependency(%q<addressable>, [">= 2.0.0"])
-  end
-end
+  s.extensions = ["ext/buffer/extconf.rb", "ext/http11_client/extconf.rb"]
 
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/fetch.rb

@@ -1,30 +1,30 @@
-require 'rubygems'
-require 'eventmachine'
-require '../lib/em-http'
-
-urls = ARGV
-if urls.size < 1
-  puts "Usage: #{$0} <url> <url> <...>"
-  exit
-end
-
-pending = urls.size
-
-EM.run do
-  urls.each do |url|
-    http = EM::HttpRequest.new(url).get
-    http.callback {
-      puts "#{url}\n#{http.response_header.status} - #{http.response.length} bytes\n"
-      puts http.response
-
-      pending -= 1
-      EM.stop if pending < 1
-    }
-    http.errback {
-      puts "#{url}\n" + http.error
-
-      pending -= 1
-      EM.stop if pending < 1
-    }
-  end
-end
+require 'rubygems'
+require 'eventmachine'
+require '../lib/em-http'
+
+urls = ARGV
+if urls.size < 1
+  puts "Usage: #{$0} <url> <url> <...>"
+  exit
+end
+
+pending = urls.size
+
+EM.run do
+  urls.each do |url|
+    http = EM::HttpRequest.new(url).get
+    http.callback {
+      puts "#{url}\n#{http.response_header.status} - #{http.response.length} bytes\n"
+      puts http.response
+
+      pending -= 1
+      EM.stop if pending < 1
+    }
+    http.errback {
+      puts "#{url}\n" + http.error
+
+      pending -= 1
+      EM.stop if pending < 1
+    }
+  end
+end

data/examples/fibered-http.rb

@@ -1,39 +1,39 @@
-require 'eventmachine'
-require 'em-http'
-require 'fiber'
-
-# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
-# while using the async EventMachine API's
-
-def async_fetch(url)
-  f = Fiber.current
-  http = EventMachine::HttpRequest.new(url).get :timeout => 10
-
-  http.callback { f.resume(http) }
-  http.errback  { f.resume(http) }
-
-  return Fiber.yield
-end
-
-EventMachine.run do
-  Fiber.new{
-
-    puts "Setting up HTTP request #1"
-    data = async_fetch('http://www.google.com/')
-    puts "Fetched page #1: #{data.response_header.status}"
-
-    puts "Setting up HTTP request #2"
-    data = async_fetch('http://www.yahoo.com/')
-    puts "Fetched page #2: #{data.response_header.status}"
-
-    EventMachine.stop
-  }.resume
-end
-
-puts "Done"
-
-#  Setting up HTTP request #1
-#  Fetched page #1: 302
-#  Setting up HTTP request #2
-#  Fetched page #2: 200
+require 'eventmachine'
+require 'em-http'
+require 'fiber'
+
+# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
+# while using the async EventMachine API's
+
+def async_fetch(url)
+  f = Fiber.current
+  http = EventMachine::HttpRequest.new(url).get :timeout => 10
+
+  http.callback { f.resume(http) }
+  http.errback  { f.resume(http) }
+
+  return Fiber.yield
+end
+
+EventMachine.run do
+  Fiber.new{
+
+    puts "Setting up HTTP request #1"
+    data = async_fetch('http://www.google.com/')
+    puts "Fetched page #1: #{data.response_header.status}"
+
+    puts "Setting up HTTP request #2"
+    data = async_fetch('http://www.yahoo.com/')
+    puts "Fetched page #2: #{data.response_header.status}"
+
+    EventMachine.stop
+  }.resume
+end
+
+puts "Done"
+
+#  Setting up HTTP request #1
+#  Fetched page #1: 302
+#  Setting up HTTP request #2
+#  Fetched page #2: 200
 #  Done