em-http-fetcher 0.1.0

data/README.rdoc

@@ -0,0 +1,89 @@
+= em-http-fetcher
+
+HTTP fetch client based on ruby EventMachne and EM-HTTP-Request
+that has configureable concurrency regardless of EM's thread pool.
+
+== Example
+
+  EM.run do
+    trap(:INT) { EM.stop }
+    fetcher = EM::HttpFetcher.new
+    fetcher.callback do |req| # req is HttpRequest instance
+      # Here is global callback block for all request
+      p "Fetch success! #{req.last_effective_url} (#{req.response.size} bytes)"
+    end
+
+    %w(
+      http://www.google.com/
+      http://heroku.com/
+      http://sourceforge.net/
+      http://github.com/
+    ).each do |url|
+      fetcher.request url
+    end
+
+    req = fetcher.request 'http://www.ruby-lang.org/'
+    req.callback do
+      # Here is appendix callback block for this request.
+      # Global callback block will also be called.
+      puts "Hello Ruby!"
+    end
+  end
+
+== Install
+
+After em-http-request *1.0.4* *or* *above* is released, you can install with gem.
+Currently em-http-fetcher depends (> 1.0.3) to prevent to install intentionally.
+
+  $ gem install em-http-fetcher
+
+=== Workaround with bundler
+
+  gem install bundler
+
+And create Gemfile to fetch develop version of em-http-request.
+
+  source "http://rubygems.org"
+  gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request.git'
+
+Then run bundle to install gems.
+
+  $ bundle
+
+Finally run your script with "bundle exec".
+
+  $ bundle exec YOUR_SCRIPT
+
+== Usage
+
+=== Options for HttpFetcher.new
+
+[:concurrency] Concurrency for all request.
+[:host_concurrency] Concurrency per host.
+[:host_request_wait] Wait specified seconds after request on each request thread.
+[(all other keys)] Pass through for HttpRequest.new
+
+=== Options for HttpFetcher#request
+
+[:uri] Target URI (String or URI object)
+[:method] Request method (get/head/put...) (default=:get)
+[(all other keys)] Pass through for HttpRequest#(get/head/put...)
+
+If first argument is not a hash, it will be treated as :uri.
+
+== Limitations
+
+* :host_concurrency is checked only for initial URI.
+  When request is redirected, number of parallel requests for
+  one host may be over host_concurrency.
+* Redirections will not work until issue #230 of em-http-request
+  is resolved; https://github.com/igrigorik/em-http-request/pull/230
+
+== License
+
+Same as Ruby 2.0 (2-clause BSDL or Ruby original license)
+
+== See Also
+
+EventMachine:: http://rubyeventmachine.com/
+EM-HTTP-Request:: https://github.com/igrigorik/em-http-request

data/em-http-fetcher.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-http-fetcher"
+
+Gem::Specification.new do |s|
+  s.name        = "em-http-fetcher"
+  s.version     = EventMachine::HttpFetcher::VERSION
+
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Tatsuki Sugiura"]
+  s.email       = ["sugi@nemui.org"]
+  s.homepage    = "http://github.com/sugi/em-http-fetcher"
+  s.summary     = "HTTP fetch client based on ruby EventMachne and EM-HTTP-Request"
+  s.description = "HTTP fetch client based on ruby EventMachne and EM-HTTP-Request that has configureable concurrency regardless of EM's thread pool."
+
+#  s.rubyforge_project = ""
+
+  s.required_ruby_version = '>= 1.9.0'
+
+  s.add_dependency "addressable", ">= 2.2.3"
+  s.add_dependency "em-http-request", "> 1.0.3"
+
+#  s.add_development_dependency "rspec"
+#  s.add_development_dependency "rake"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end
+

data/lib/em-http-fetcher.rb

@@ -0,0 +1 @@
+require 'em/http-fetcher/fetcher.rb'

data/lib/em/http-fetcher/fetcher.rb

@@ -0,0 +1,163 @@
+#!/usr/bin/env ruby
+require 'eventmachine'
+require 'em/pool'
+require 'em-http-request'
+require 'addressable/uri'
+
+module EventMachine
+  class HttpFetcher
+    VERSION = "0.1.0"
+
+    class RequestPool
+      def initialize(total_size, host_resource_size, host_reuse_wait = 0, opts = {})
+        super()
+        @total_size         = total_size
+        @host_reuse_wait    = host_reuse_wait
+        @host_resource_size = host_resource_size
+
+        @total_queue        = EM::Queue.new
+        total_size.times { @total_queue.push true }
+        @host_pools         = Hash.new {|h, k|
+          pool = EM::Pool.new
+          def pool.add item
+            super
+            @removed.delete item
+          end
+          host_resource_size.times {
+            pool.add EM::HttpRequest.new(k)
+          }
+          h[k] = { pool: pool, last_used: Time.now }
+        }
+        run
+      end
+
+      def perform(host, &b)
+        @host_pools[host][:pool].perform do |conn|
+          df = nil
+          @total_queue.pop do |tqi|
+            @host_pools[host][:last_used] = Time.now
+            @host_pools[host][:pool].remove conn
+            rq = proc { |req|
+              @total_queue.push tqi
+              lurl = req.last_effective_url
+              unless "#{lurl.scheme}://#{lurl.host}" == host
+                # Connection has been redirected another server.
+                # Re-create connection instance.
+                conn = EM::HttpRequest.new(host)
+              end
+
+              if @host_reuse_wait > 0
+                EM.add_timer(@host_reuse_wait) {
+                  @host_pools[host][:pool].add conn
+                }
+              else
+                @host_pools[host][:pool].add conn
+              end
+            }
+            work = EM::Callback(&b)
+            df = work.call(conn)
+            df.callback(&rq)
+            df.errback(&rq)
+            df
+          end
+          df
+        end
+      end
+
+      def run
+        # cleanup host pool timer
+        EM.add_periodic_timer(10) do
+          hrsize = @host_resource_size
+          @host_pools.each do |host, info|
+            info[:pool].instance_eval { @resources.size < hrsize } and next
+            info[:last_used].to_i > Time.now.to_i - 5 * 60 and next
+            @host_pools.delete host
+          end
+        end
+      end
+    end
+
+    def initialize(opts = {})
+      @concurrency       = opts[:concurrency]       || 40
+      @host_concurrency  = opts[:host_concurrency]  || 2
+      @host_request_wait = opts[:host_request_wait] || 0.2
+      @request_pool      = nil
+      @default_callbacks = []
+      @default_errbacks  = []
+      @req_opts = {}.merge(opts)
+      @req_opts.delete :concurrency
+      @req_opts.delete :host_concurrency
+      @req_opts.delete :host_request_wait
+    end
+
+    def request_pool
+      @request_pool ||= RequestPool.new(@concurrency, @host_concurrency, @host_request_wait, @req_opts)
+    end
+
+    def callback(&block)
+      @default_callbacks << block
+    end
+
+    def errback(&block)
+     @default_errbacks << block
+    end
+
+    def request(*args)
+      if args.first.kind_of? Hash
+        opts = args[0]
+        uri = opts.delete(:uri)
+      else
+        uri = args.first
+        opts = args[1].kind_of?(Hash) ? args[1] : {}
+      end
+
+      uri.kind_of?(Addressable::URI) or
+        uri = Addressable::URI.parse(uri.to_s)
+      opts = {
+        :keepalive => true,
+        :redirects => 20,
+        :path => uri.path || '/',
+      }.merge(opts)
+      method = opts.delete(:method) || :get
+      uri.query and otps[:query] = uri.query
+
+      df = nil
+      request_pool.perform("#{uri.scheme}://#{uri.host}") do |conn|
+        df = req = conn.__send__(method, opts)
+        @default_callbacks.each do |cb|
+          req.callback(&cb)
+        end
+        @default_errbacks.each do |cb|
+          req.errback(&cb)
+        end
+        req
+      end
+      df
+    end
+  end
+end
+
+
+if __FILE__ == $0
+  trap(:INT) { EM.stop }
+  EM.run do
+    r = EM::HttpFetcher.new
+    r.callback do |req|
+      p [:success, req.last_effective_url, req.response.size]
+    end
+    r.errback do |req|
+      p [:err, req.last_effective_url, req.response.size]
+    end
+
+    ARGF.each { |line|
+      line.chomp!
+      line or next
+      req = r.request(line)
+      if line == 'http://www.yahoo.co.jp/'
+        req.callback do
+          p :yahoo!
+        end
+      end
+    }
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: em-http-fetcher
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Tatsuki Sugiura
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.3
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+description: HTTP fetch client based on ruby EventMachne and EM-HTTP-Request that
+  has configureable concurrency regardless of EM's thread pool.
+email:
+- sugi@nemui.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.rdoc
+- em-http-fetcher.gemspec
+- lib/em-http-fetcher.rb
+- lib/em/http-fetcher/fetcher.rb
+homepage: http://github.com/sugi/em-http-fetcher
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: HTTP fetch client based on ruby EventMachne and EM-HTTP-Request
+test_files: []