z-http-request 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ab688a2d990741d66ecc79a6d7f7fcb5908e60ad
+  data.tar.gz: 81fc6baa002d22c532883c4a1f8986d1170ffc82
+SHA512:
+  metadata.gz: 0aa272059eb771c1451b67ba9a85561cf54a0aa8b6dba3933afa0858d6b662f03d09b508b223fd62fa3af1b0d472514d5c7ee89b4458a7b4f835f649faf60ef2
+  data.tar.gz: a04b808c207ef7f2e34c80233b25d2d42f21b45cd8981ae5dac176cc661c27bd4867f746c0e0159dcf7f2db34e774cedbe9cb8ed5a49d5bbacfc2384e10c0d36

data/.gitignore

@@ -0,0 +1,10 @@
+*.bundle
+*.o
+.idea
+.jbundler/
+Gemfile.lock
+Makefile
+gems/
+misc
+mkmf.log
+vendor/

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--require spec/spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+jruby

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+- jruby
+notifications:
+  email: false
+  hipchat:
+    rooms:
+      secure: QEn6sr/TryfI6Kjy39oXnNwfFxvHXbWNk413X0Ocno7FVbddu1DpN3+8FgfUTL7GTCxKyV4cKni9Zp+hiLrlUO15zk3JDZKzLBW24Ie0VCmhhYQAQWuVYatdo2x16LzyyFijNpvNlIjoRoifzIfF7YMNszgy4FxJ4ThFPp34jIY=

data/Gemfile

@@ -0,0 +1,17 @@
+source 'http://rubygems.org'
+
+gemspec
+
+gem 'ruby-debug'
+gem 'liquid-ext'
+gem 'zmachine', :path => '../zmachine'
+
+group :benchmark do
+  gem 'excon'
+  gem 'httparty'
+  gem 'rest-client'
+  gem 'sinatra'
+  gem 'streamly_ffi'
+  gem 'tach', '>= 0.0.8'
+  gem 'typhoeus'
+end

data/README.md

@@ -0,0 +1,38 @@
+# Z-HTTP-Request
+
+This is a fork of Ilya Gregoriks em-http-request, ported to
+[ZMachine](https://github.com/liquidm/zmachine). Furthermore stuff like
+socksify has been removed as it is not needed for our simple http client use
+case.
+
+[![Build Status](https://travis-ci.org/liquidm/z-http-request.png)](https://travis-ci.org/liquidm/z-http-request)
+[![Code Climate](https://codeclimate.com/github/liquidm/z-http-request.png)](https://codeclimate.com/github/liquidm/z-http-request)
+[![Dependency Status](https://gemnasium.com/liquidm/z-http-request.png)](https://gemnasium.com/liquidm/z-http-request)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'z-http-request'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install z-http-request
+
+## Usage
+
+Z-HTTP-Request is mostly API compatible with em-http-request and EventMachine.
+Replace `EM` / `EventMachine` with `ZMachine` in your code and it should work
+out of the box.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/setup"
+require "bundler/gem_tasks"
+require "liquid/tasks"

data/benchmarks/clients.rb

@@ -0,0 +1,170 @@
+$: << './benchmarks'
+require 'server'
+
+require 'excon'
+require 'httparty'
+require 'net/http'
+require 'open-uri'
+require 'rest_client'
+require 'tach'
+require 'typhoeus'
+
+url = 'http://127.0.0.1:9292/data/10000'
+
+with_server do
+  Tach.meter(100) do
+
+    tach('curb (persistent)') do |n|
+      curb = Curl::Easy.new
+
+      n.times do
+        curb.url = url
+        curb.http_get
+        curb.body_str
+      end
+    end
+
+    tach('z-http-request') do |n|
+      ZMachine.run {
+        count = 0
+        error = 0
+
+        n.times do
+          http = ZMachine::HttpRequest.new(url).get
+
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+        end
+      }
+    end
+
+    tach('z-http-request (persistent)') do |n|
+      ZMachine.run {
+        count = 0
+        error = 0
+
+        conn = ZMachine::HttpRequest.new(url)
+
+        n.times do
+          http = conn.get :keepalive => true
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+        end
+      }
+    end
+
+    tach('Excon') do
+      Excon.get(url).body
+    end
+
+    excon = Excon.new(url)
+    tach('Excon (persistent)') do
+      excon.request(:method => 'get').body
+    end
+
+    tach('HTTParty') do
+      HTTParty.get(url).body
+    end
+
+    uri = Addressable::URI.parse(url)
+    tach('Net::HTTP') do
+      Net::HTTP.start(uri.host, uri.port) {|http| http.get(uri.path).body }
+    end
+
+    uri = Addressable::URI.parse(url)
+    Net::HTTP.start(uri.host, uri.port) do |http|
+      tach('Net::HTTP (persistent)') do
+        http.get(uri.path).body
+      end
+    end
+
+    tach('open-uri') do
+      open(url).read
+    end
+
+    tach('RestClient') do
+      RestClient.get(url)
+    end
+
+    streamly = StreamlyFFI::Connection.new
+    tach('StreamlyFFI (persistent)') do
+      streamly.get(url)
+    end
+
+    tach('Typhoeus') do |n|
+        hydra = Typhoeus::Hydra.new( max_concurrency: 8 )
+        hydra.disable_memoization
+        count = 0
+        error = 0
+        n.times {
+            req = Typhoeus::Request.new( url )
+            req.on_complete do |res|
+                count += 1
+                error += 1 if !res.success?
+                p [count, error] if count == n
+
+            end
+            hydra.queue( req )
+        }
+        hydra.run
+    end
+
+  end
+end
+
+
+#+------------------------------+-----------+
+#| tach                         | total     |
+#+------------------------------+-----------+
+#| z-http-request (persistent) | 0.145512  |
+#+------------------------------+-----------+
+#| Excon                        | 0.181564  |
+#+------------------------------+-----------+
+#| RestClient                   | 0.253127  |
+#+------------------------------+-----------+
+#| Net::HTTP                    | 0.294412  |
+#+------------------------------+-----------+
+#| HTTParty                     | 0.305397  |
+#+------------------------------+-----------+
+#| open-uri                     | 0.307007  |
+#+------------------------------+-----------+
+#| Net::HTTP (persistent)       | 0.313716  |
+#+------------------------------+-----------+
+#| Typhoeus                     | 0.514725  |
+#+------------------------------+-----------+
+#| curb (persistent)            | 3.981700  |
+#+------------------------------+-----------+
+#| StreamlyFFI (persistent)     | 3.989063  |
+#+------------------------------+-----------+
+#| Excon (persistent)           | 4.018761  |
+#+------------------------------+-----------+
+#| z-http-request              | 15.025291 |
+#+------------------------------+-----------+

data/benchmarks/em-excon.rb

@@ -0,0 +1,87 @@
+$: << './benchmarks'
+require 'server'
+
+url = 'http://127.0.0.1/10k.html'
+
+with_server do
+  Tach.meter(100) do
+
+    excon = Excon.new(url)
+    tach('Excon (persistent)') do
+      excon.request(:method => 'get').body
+    end
+
+    tach('Excon') do
+      Excon.get(url).body
+    end
+
+    tach('z-http-request') do |n|
+      ZMachine.run {
+        count = 0
+        error = 0
+        n.times do
+          ZMachine.next_tick do
+            http = ZMachine::HttpRequest.new(url, :connect_timeout => 1).get
+
+            http.callback {
+              count += 1
+              if count == n
+                p [count, error]
+                ZMachine.stop
+              end
+            }
+
+            http.errback {
+              count += 1
+              error += 1
+              if count == n
+                p [count, error]
+                ZMachine.stop
+              end
+            }
+          end
+        end
+      }
+    end
+
+    tach('z-http-request (persistent)') do |n|
+      ZMachine.run {
+        count = 0
+        error = 0
+        conn = ZMachine::HttpRequest.new(url)
+
+        n.times do
+          http = conn.get :keepalive => true
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              ZMachine.stop
+            end
+          }
+        end
+      }
+    end
+  end
+end
+
+# +------------------------------+----------+
+# | tach                         | total    |
+# +------------------------------+----------+
+# | z-http-request (persistent) | 0.018133 |
+# +------------------------------+----------+
+# | Excon (persistent)           | 0.023975 |
+# +------------------------------+----------+
+# | Excon                        | 0.032877 |
+# +------------------------------+----------+
+# | z-http-request              | 0.042891 |
+# +------------------------------+----------+

data/benchmarks/em-profile.txt

@@ -0,0 +1,65 @@
+Total: 143 samples
+      67  46.9%  46.9%      109  76.2% Kernel#gem_original_require
+      17  11.9%  58.7%       17  11.9% garbage_collector
+      12   8.4%  67.1%       12   8.4% ZMachine.connect_server
+      11   7.7%  74.8%       11   7.7% Dir.[]
+      11   7.7%  82.5%       11   7.7% File.file?
+       4   2.8%  85.3%        4   2.8% Kernel#eval
+       3   2.1%  87.4%       16  11.2% Enumerable#find
+       3   2.1%  89.5%        3   2.1% Regexp#initialize
+       2   1.4%  90.9%       16  11.2% ZMachine.run_machine
+       2   1.4%  92.3%        2   1.4% Regexp#match
+       1   0.7%  93.0%        1   0.7% FFI::DynamicLibrary.open
+       1   0.7%  93.7%        1   0.7% Hash#values
+       1   0.7%  94.4%        7   4.9% Kernel#load
+       1   0.7%  95.1%        3   2.1% MIME::Type#initialize
+       1   0.7%  95.8%        1   0.7% MatchData#captures
+       1   0.7%  96.5%       13   9.1% Module#bind_connect
+       1   0.7%  97.2%       17  11.9% Module#searcher
+       1   0.7%  97.9%        1   0.7% String#downcase
+       1   0.7%  98.6%        1   0.7% String#scan
+       1   0.7%  99.3%        1   0.7% String#strip
+       1   0.7% 100.0%        1   0.7% TCPSocket#initialize
+       0   0.0% 100.0%        1   0.7% Array#map
+       0   0.0% 100.0%       14   9.8% Array#reverse_each
+       0   0.0% 100.0%       14   9.8% Class#from_gems_in
+       0   0.0% 100.0%       14   9.8% Class#from_installed_gems
+       0   0.0% 100.0%       14   9.8% Class#load
+       0   0.0% 100.0%        1   0.7% Class#parse
+       0   0.0% 100.0%        1   0.7% Class#simplified
+       0   0.0% 100.0%        6   4.2% Enumerable#each_with_index
+       0   0.0% 100.0%        1   0.7% Excon::Connection#connect
+       0   0.0% 100.0%        1   0.7% Excon::Connection#request
+       0   0.0% 100.0%        1   0.7% Excon::Connection#socket
+       0   0.0% 100.0%        1   0.7% FFI::Library#ffi_lib
+       0   0.0% 100.0%       10   7.0% GZMachine::GemPathSearcher#find
+       0   0.0% 100.0%        1   0.7% GZMachine::GemPathSearcher#find_active
+       0   0.0% 100.0%       16  11.2% GZMachine::GemPathSearcher#init_gemspecs
+       0   0.0% 100.0%       16  11.2% GZMachine::GemPathSearcher#initialize
+       0   0.0% 100.0%       11   7.7% GZMachine::GemPathSearcher#matching_file?
+       0   0.0% 100.0%       11   7.7% GZMachine::GemPathSearcher#matching_files
+       0   0.0% 100.0%       14   9.8% GZMachine::SourceIndex#load_gems_in
+       0   0.0% 100.0%       14   9.8% GZMachine::SourceIndex#refresh!
+       0   0.0% 100.0%        1   0.7% GZMachine::SourceIndex#search
+       0   0.0% 100.0%        1   0.7% HttpOptions#initialize
+       0   0.0% 100.0%        1   0.7% HttpOptions#set_uri
+       0   0.0% 100.0%       14   9.8% Integer#times
+       0   0.0% 100.0%        1   0.7% Kernel#loop
+       0   0.0% 100.0%      109  76.2% Kernel#require
+       0   0.0% 100.0%        1   0.7% Module#activate
+       0   0.0% 100.0%       13   9.1% Module#connect
+       0   0.0% 100.0%        1   0.7% Module#get
+       0   0.0% 100.0%        2   1.4% Module#load_full_rubygems_library
+       0   0.0% 100.0%        5   3.5% Module#loaded_path?
+       0   0.0% 100.0%       16  11.2% Module#meter
+       0   0.0% 100.0%       16  11.2% Module#run
+       0   0.0% 100.0%       16  11.2% Module#source_index
+       0   0.0% 100.0%       30  21.0% Module#try_activate
+       0   0.0% 100.0%       17  11.9% Object#with_server
+       0   0.0% 100.0%        2   1.4% Regexp.union
+       0   0.0% 100.0%        1   0.7% TCPSocket.open
+       0   0.0% 100.0%       16  11.2% Tach::Meter#initialize
+       0   0.0% 100.0%       16  11.2% Tach::Meter#run_tach
+       0   0.0% 100.0%        1   0.7% URI::Parser#initialize
+       0   0.0% 100.0%        1   0.7% URI::Parser#initialize_regexp
+       0   0.0% 100.0%        7   4.9% YAML::EngineManager#yamler=

data/benchmarks/server.rb

@@ -0,0 +1,48 @@
+require 'excon'
+require 'httparty'
+require 'net/http'
+require 'open-uri'
+require 'rest_client'
+require 'tach'
+require 'typhoeus'
+require 'sinatra/base'
+require 'streamly_ffi'
+require 'curb'
+
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'z-http')
+
+module Benchmark
+  class Server < Sinatra::Base
+
+    def self.run
+      Rack::Handler::WEBrick.run(
+        Benchmark::Server.new,
+        :Port => 9292,
+        :AccessLog => [],
+        :Logger => WEBrick::Log.new(nil, WEBrick::Log::ERROR)
+      )
+    end
+
+    get '/data/:amount' do |amount|
+      'x' * amount.to_i
+    end
+
+  end
+end
+
+def with_server(&block)
+  pid = Process.fork do
+     Benchmark::Server.run
+  end
+  loop do
+    sleep(1)
+    begin
+      #Excon.get('http://localhost:9292/api/foo')
+      break
+    rescue
+    end
+  end
+  yield
+ensure
+  Process.kill(9, pid)
+end

data/examples/.gitignore

@@ -0,0 +1 @@
+twitter_auth.rb

data/examples/digest_auth/client.rb

@@ -0,0 +1,25 @@
+$: << 'lib' << '../../lib'
+
+require 'z-http'
+require 'z-http/middleware/digest_auth'
+
+digest_config = {
+  :username => 'digest_username',
+  :password => 'digest_password'
+}
+
+ZMachine.run do
+
+  conn_handshake = ZMachine::HttpRequest.new('http://localhost:3000')
+  http_handshake = conn_handshake.get
+
+  http_handshake.callback do
+    conn = ZMachine::HttpRequest.new('http://localhost:3000')
+    conn.use ZMachine::Middleware::DigestAuth, http_handshake.response_header['WWW_AUTHENTICATE'], digest_config
+    http = conn.get
+    http.callback do
+      puts http.response
+      ZMachine.stop
+    end
+  end
+end

data/examples/digest_auth/server.rb

@@ -0,0 +1,28 @@
+require 'webrick'
+
+include WEBrick
+
+config = { :Realm => 'DigestAuth_REALM' }
+
+htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file'
+htdigest.set_passwd config[:Realm], 'digest_username', 'digest_password'
+htdigest.flush
+
+config[:UserDB] = htdigest
+
+digest_auth = WEBrick::HTTPAuth::DigestAuth.new config
+
+class TestServlet < HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    @options[0][:authenticator].authenticate req, res
+    res.body = "You are authenticated to see the super secret data\n"
+  end
+end
+
+s = HTTPServer.new(:Port => 3000)
+s.mount('/', TestServlet, {:authenticator => digest_auth})
+trap("INT") do
+  File.delete('my_password_file')
+  s.shutdown
+end
+s.start

data/examples/fetch.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'ZMachine'
+require '../lib/z-http'
+
+urls = ARGV
+if urls.size < 1
+  puts "Usage: #{$0} <url> <url> <...>"
+  exit
+end
+
+pending = urls.size
+
+ZMachine.run do
+  urls.each do |url|
+    http = ZMachine::HttpRequest.new(url).get
+    http.callback {
+      puts "#{url}\n#{http.response_header.status} - #{http.response.length} bytes\n"
+      puts http.response
+
+      pending -= 1
+      ZMachine.stop if pending < 1
+    }
+    http.errback {
+      puts "#{url}\n" + http.error
+
+      pending -= 1
+      ZMachine.stop if pending < 1
+    }
+  end
+end

data/examples/fibered-http.rb

@@ -0,0 +1,51 @@
+$: << 'lib' << '../lib'
+
+require 'ZMachine'
+require 'z-http'
+require 'fiber'
+
+# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
+# while using the async ZMachine API's
+
+def async_fetch(url)
+  f = Fiber.current
+  http = ZMachine::HttpRequest.new(url, :connect_timeout => 10, :inactivity_timeout => 20).get
+
+  http.callback { f.resume(http) }
+  http.errback  { f.resume(http) }
+
+  Fiber.yield
+
+  if http.error
+    p [:HTTP_ERROR, http.error]
+  end
+
+  http
+end
+
+ZMachine.run do
+  Fiber.new{
+
+    puts "Setting up HTTP request #1"
+    data = async_fetch('http://0.0.0.0/')
+    puts "Fetched page #1: #{data.response_header.status}"
+
+    puts "Setting up HTTP request #2"
+    data = async_fetch('http://www.yahoo.com/')
+    puts "Fetched page #2: #{data.response_header.status}"
+
+    puts "Setting up HTTP request #3"
+    data = async_fetch('http://non-existing.domain/')
+    puts "Fetched page #3: #{data.response_header.status}"
+
+    ZMachine.stop
+  }.resume
+end
+
+puts "Done"
+
+#  Setting up HTTP request #1
+#  Fetched page #1: 302
+#  Setting up HTTP request #2
+#  Fetched page #2: 200
+#  Done

data/examples/multi.rb

@@ -0,0 +1,25 @@
+$: << '../lib' << 'lib'
+
+require 'ZMachine'
+require 'z-http'
+
+ZMachine.run {
+  multi = ZMachine::MultiRequest.new
+
+  reqs = [
+    'http://google.com/',
+    'http://google.ca:81/'
+  ]
+
+  reqs.each_with_index do |url, idx|
+    http = ZMachine::HttpRequest.new(url, :connect_timeout => 1)
+    req = http.get
+    multi.add idx, req
+  end
+
+  multi.callback  do
+    p multi.responses[:callback].size
+    p multi.responses[:errback].size
+    ZMachine.stop
+  end
+}

data/examples/oauth-tweet.rb

@@ -0,0 +1,35 @@
+$: << 'lib' << '../lib'
+
+require 'z-http'
+require 'z-http/middleware/oauth'
+require 'z-http/middleware/json_response'
+
+require 'pp'
+
+OAuthConfig = {
+  :consumer_key     => '',
+  :consumer_secret  => '',
+  :access_token     => '',
+  :access_token_secret => ''
+}
+
+ZMachine.run do
+  # automatically parse the JSON response into a Ruby object
+  ZMachine::HttpRequest.use ZMachine::Middleware::JSONResponse
+
+  # sign the request with OAuth credentials
+  conn = ZMachine::HttpRequest.new('http://api.twitter.com/1/statuses/home_timeline.json')
+  conn.use ZMachine::Middleware::OAuth, OAuthConfig
+
+  http = conn.get
+  http.callback do
+    pp http.response
+    ZMachine.stop
+  end
+
+  http.errback do
+    puts "Failed retrieving user stream."
+    pp http.response
+    ZMachine.stop
+  end
+end