concurrent_downloader 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--format nested
+--color
+--profile

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in concurrent_downloader.gemspec
+gemspec
+
+gem 'puma'
+gem 'rspec'
+gem 'yajl-ruby'
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Oliver Clarke
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# ConcurrentDownloader
+
+Concurrently download http requests in a synchronous environment. Basically this is an em-http-request wrapper that slots into a normal ruby script.
+
+This would be useful inside a rails environment or a rake task.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'concurrent_downloader'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install concurrent_downloader
+
+## Usage
+
+Check out spec/lib/concurrent_downloader_spec.rb for lots of usage examples. Here is one:
+
+      queue = []
+      100.times do |i|
+        queue << {
+          :method => "post",
+          :path   => "/test"
+        }
+      end
+
+      responses = []
+      ConcurrentDownloader.process_queue!(queue, :host => "http://localhost", :concurrent_downloads => 10) do |queue_item, response|
+        if response.status == 200
+          responses << response
+        end
+      end
+
+      # if all responses are 200
+      responses.size.should == 100
+
+The test suite passes in both 1.8.7 and 1.9.3
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Contact
+
+Oliver Clarke <ollie@clarketus.net>
+

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/concurrent_downloader.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/concurrent_downloader/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Oliver Clarke"]
+  gem.email         = ["ollie@clarketus.net"]
+  gem.description   = %q{Concurrently download http requests in a synchronous environment.}
+  gem.summary       = %q{A basic em-http-request wrapper that slots into a normal ruby script.}
+  gem.homepage      = "https://github.com/clarketus/concurrent_downloader"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "concurrent_downloader"
+  gem.require_paths = ["lib"]
+  gem.version       = ConcurrentDownloader::VERSION
+
+  gem.add_dependency('eventmachine')
+  gem.add_dependency('em-http-request')
+end
+

data/lib/concurrent_downloader.rb

@@ -0,0 +1,53 @@
+require 'logger'
+
+require 'eventmachine'
+require 'em-http-request'
+
+require 'concurrent_downloader/response'
+require 'concurrent_downloader/processor'
+
+module ConcurrentDownloader
+
+  class << self
+    def process_queue!(queue, options={}, &block)
+      Processor.process_queue!(queue, options, &block)
+    end
+
+    def logger
+      @logger ||= Logger.new('/dev/null')
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+
+  class ConnectionError < StandardError
+    attr_reader \
+      :queue_item,
+      :downloader_id
+
+    def initialize(queue_item, downloader_id)
+      @queue_item = queue_item
+      @downloader_id = downloader_id
+
+      super "There was a connection error: #{@queue_item[:method].upcase} #{@queue_item[:path]}"
+    end
+  end
+
+  class DownloadError < StandardError
+    attr_reader \
+      :queue_item,
+      :response,
+      :downloader_id
+
+    def initialize(queue_item, response, downloader_id)
+      @queue_item = queue_item
+      @response = response
+      @downloader_id = downloader_id
+
+      super "There was a download error: #{@queue_item[:method].upcase} #{@queue_item[:path]}: #{@response.status}"
+    end
+  end
+end
+

data/lib/concurrent_downloader/processor.rb

@@ -0,0 +1,103 @@
+module ConcurrentDownloader
+  class Processor
+
+    class << self
+      def process_queue!(queue, options={}, &block)
+        new.process_queue!(queue, options, &block)
+      end
+    end
+
+    def process_queue!(queue, options={}, &block)
+      @queue                = queue || []
+      @host                 = options[:host]
+
+      @error_limit          = options[:error_limit] || 0
+      @concurrent_downloads = options[:concurrent_downloads] || 1
+      @connect_timeout      = options[:connect_timeout] || 5
+      @inactivity_timeout   = options[:inactivity_timeout] || 10
+
+      @error_count          = 0
+      @error_limit_passed   = false
+      @response_block       = block
+
+      EM.run do
+        @concurrent_downloads.times do |downloader_id|
+          recursive_download(downloader_id)
+        end
+      end
+    end
+
+    private
+
+    def recursive_download(downloader_id)
+      if queue_item = @queue.pop
+        if queue_item.is_a?(String)
+          queue_item = {:path => queue_item}
+        end
+
+        queue_item[:method] ||= "get"
+
+        method  = queue_item[:method]
+        path    = queue_item[:path]
+        body    = queue_item[:body]
+        head    = queue_item[:head] || {}
+
+        head = head.merge(:downloader_id => downloader_id)
+
+        ConcurrentDownloader.logger.info "#{downloader_id} => #{method} #{path}"
+
+        connection = EM::HttpRequest.new @host,
+          :connect_timeout    => @connect_timeout,
+          :inactivity_timeout => @inactivity_timeout
+
+        request = connection.send method,
+          :path => path,
+          :body => body,
+          :head => head
+
+        request.callback do |request|
+          response = Response.new \
+            :status   => request.response_header.status,
+            :headers  => Hash[request.response_header],
+            :body     => request.response
+
+          if !@response_block.call(queue_item, response)
+            handle_error DownloadError.new(queue_item, response, downloader_id)
+          end
+        end
+
+        request.errback do |request|
+          handle_error ConnectionError.new(queue_item, downloader_id)
+        end
+
+        [:callback, :errback].each do |meth|
+          request.send(meth) do
+            recursive_download(downloader_id)
+          end
+        end
+      else
+        @concurrent_downloads -= 1
+        if @concurrent_downloads == 0
+          EM.stop
+
+          if @error_limit_passed
+            raise @last_error
+          end
+        end
+      end
+    end
+
+    def handle_error(error)
+      ConcurrentDownloader.logger.info "#{error.downloader_id} => #{error.class}: #{error.message}"
+      @last_error = error
+
+      if @error_count < @error_limit
+        @error_count += 1
+        @queue << error.queue_item
+      else
+        @error_limit_passed = true
+        @queue = []
+      end
+    end
+  end
+end

data/lib/concurrent_downloader/response.rb

@@ -0,0 +1,14 @@
+module ConcurrentDownloader
+  class Response
+    attr_reader \
+      :body,
+      :status,
+      :headers
+
+    def initialize(data={})
+      @body     = data[:body]
+      @status   = data[:status]
+      @headers  = data[:headers]
+    end
+  end
+end

data/lib/concurrent_downloader/version.rb

@@ -0,0 +1,3 @@
+module ConcurrentDownloader
+  VERSION = "0.0.1"
+end

data/spec/lib/concurrent_downloader_spec.rb

@@ -0,0 +1,253 @@
+require 'spec_helper'
+
+describe ConcurrentDownloader do
+
+  context "Basic operation" do
+    it "should process a basic queue" do
+      queue = []
+      queue << "/test"
+
+      responses = []
+      ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri) do |queue_item, response|
+        responses << response
+      end
+
+      responses.size.should == 1
+      response = responses.first
+
+      response.should be_a(ConcurrentDownloader::Response)
+      response.status.should == 200
+      response.headers.should == {"CONNECTION"=>"close", "CONTENT_LENGTH"=>"82"}
+      Yajl.load(response.body).should == {
+        "path"          => "/test",
+        "body"          => nil,
+        "downloader_id" => "0",
+        "method"        => "GET",
+        "test_header"   => nil
+      }
+    end
+
+    it "should allow setting of specific request options" do
+      queue = []
+      queue << {
+        :method => "post",
+        :path   => "/test",
+        :body   => {"test_param_key" => "test_param_value"},
+        :head   => {"test_header_key" => "test_header_value"}
+      }
+
+      responses = []
+      ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri) do |queue_item, response|
+        responses << response
+      end
+
+      responses.size.should == 1
+      response = responses.first
+
+      response.status.should == 200
+      response.headers.should == {"CONNECTION"=>"close", "CONTENT_LENGTH"=>"127"}
+      Yajl.load(response.body).should == {
+        "path"          => "/test",
+        "body"          => "test_param_key=test_param_value",
+        "downloader_id" => "0",
+        "method"        => "POST",
+        "test_header"   => "test_header_value"
+      }
+    end
+  end
+
+  context "large queues" do
+    it "should process a large queue" do
+      queue = []
+      100.times do |i|
+        queue << {
+          :method => "post",
+          :path   => "/test",
+          :body   => {"request_number" => "%03d" % i},
+        }
+      end
+
+      responses = []
+      ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri) do |queue_item, response|
+        responses << response
+      end
+
+      responses.size.should == 100
+
+      responses.reverse.each_with_index do |response, index|
+        response.status.should == 200
+        response.headers.should == {"CONNECTION"=>"close", "CONTENT_LENGTH"=>"99"}
+        Yajl.load(response.body).should == {
+          "path"          => "/test",
+          "body"          => "request_number=#{"%03d" % index}",
+          "downloader_id" => "0",
+          "method"        => "POST",
+          "test_header"   => nil
+        }
+      end
+    end
+
+    it "should allow concurrent requests" do
+      queue = []
+      100.times do |i|
+        queue << {
+          :method => "post",
+          :path   => "/test"
+        }
+      end
+
+      responses = []
+      ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri, :concurrent_downloads => 10) do |queue_item, response|
+        responses << response
+      end
+
+      downloader_ids = []
+
+      responses.size.should == 100
+      responses.each do |response|
+        response.status.should == 200
+        response.headers.should == {"CONNECTION"=>"close", "CONTENT_LENGTH"=>"81"}
+        body = Yajl.load(response.body)
+        body["path"].should == "/test"
+        body["method"].should == "POST"
+
+        downloader_ids << body["downloader_id"].to_i
+      end
+
+      (0..9).each{|i| downloader_ids.should include(i) }
+    end
+  end
+
+  context "Error handling" do
+    it "should raise when there is a connection error" do
+      port = find_available_port
+
+      queue = []
+      queue << "/test"
+
+      exception = nil
+      begin
+        ConcurrentDownloader.process_queue!(queue, :host => "http://127.0.0.1:#{port}")
+      rescue => e
+        exception = e
+      end
+
+      exception.should_not be_nil
+      exception.should be_a(ConcurrentDownloader::ConnectionError)
+      exception.message.should == "There was a connection error: GET /test"
+      exception.downloader_id.should == 0
+      exception.queue_item.should == {
+        :path => "/test",
+        :method => "get"
+      }
+    end
+
+    it "should raise when there is an inactivity timeout" do
+      begin
+        server = TCPServer.new('127.0.0.1', 0)
+        port = server.addr[1]
+
+        queue = []
+        queue << "/test"
+
+        exception = nil
+        begin
+          ConcurrentDownloader.process_queue!(queue, :host => "http://127.0.0.1:#{port}", :inactivity_timeout => 1)
+        rescue => e
+          exception = e
+        end
+
+        exception.should_not be_nil
+        exception.should be_a(ConcurrentDownloader::ConnectionError)
+        exception.message.should == "There was a connection error: GET /test"
+      ensure
+        server.close if server
+      end
+    end
+
+    it "should raise a download error when request block returns false" do
+      queue = []
+      queue << "/test"
+
+      exception = nil
+
+      begin
+        ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri) do |queue_item, response|
+          false
+        end
+      rescue => e
+        exception = e
+      end
+
+      exception.should_not be_nil
+      exception.should be_a(ConcurrentDownloader::DownloadError)
+      exception.message.should == "There was a download error: GET /test: 200"
+      exception.response.should be_a(ConcurrentDownloader::Response)
+      exception.downloader_id.should == 0
+      exception.queue_item.should == {
+        :path => "/test",
+        :method => "get"
+      }
+    end
+
+    it "should retry downloading if an error occurs and the limit is not reached" do
+      queue = []
+      10.times do
+        queue << "/test"
+      end
+
+      responses = []
+
+      count = 0
+      ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri, :error_limit => 20) do |queue_item, response|
+        count += 1
+        if count > 20
+          responses << response
+        end
+
+        count > 20
+      end
+
+      count.should == 30 # 30 total requests
+      responses.size.should == 10 # 10 succeeded
+      responses.each do |response|
+        response.status.should == 200
+      end
+    end
+
+    it "should raise an error if the limit is reached and cancel all downloads in the queue" do
+      queue = []
+      10.times do
+        queue << "/test"
+      end
+
+      responses = []
+      exception = nil
+
+      begin
+        count = 0
+        ConcurrentDownloader.process_queue!(queue, :host => $mock_host_uri, :error_limit => 20) do |queue_item, response|
+          count += 1
+          if count < 5
+            responses << response
+          end
+
+          count < 5
+        end
+      rescue => e
+        exception = e
+      end
+
+      exception.should_not be_nil
+      exception.should be_a(ConcurrentDownloader::DownloadError)
+      exception.message.should == "There was a download error: GET /test: 200"
+
+      count.should == 25 # 25 total requests
+      responses.size.should == 4 # 4 succeeded
+      responses.each do |response|
+        response.status.should == 200
+      end
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,26 @@
+require 'concurrent_downloader'
+require 'puma'
+require 'yajl'
+
+Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
+
+include MockServerHelper
+
+RSpec.configure do |config|
+  config.before :suite do
+    # ConcurrentDownloader.logger = Logger.new($stdout)
+
+    $mock_host_uri = start_mock_server do |request|
+      body = {
+        :path          => request.path,
+        :body          => (request.post? ? request.body.read : nil),
+        :method        => request.request_method,
+        :test_header   => request.env["HTTP_TEST_HEADER_KEY"],
+        :downloader_id => request.env["HTTP_DOWNLOADER_ID"]
+      }
+
+      [200, [], [Yajl.dump(body)]]
+    end
+  end
+end
+

data/spec/support/mock_server_helper.rb

@@ -0,0 +1,26 @@
+module MockServerHelper
+  def start_mock_server(&block)
+    app = lambda {|env|
+      request = Rack::Request.new(env)
+      block.call(request)
+    }
+
+    port = find_available_port
+
+    Thread.new do
+      Puma::Server.new(app).tap do |s|
+        s.add_tcp_listener '127.0.0.1', port
+      end.run.join
+    end
+
+    URI.parse "http://127.0.0.1:#{port}"
+  end
+
+  def find_available_port
+    server = TCPServer.new('127.0.0.1', 0)
+    server.addr[1]
+  ensure
+    server.close if server
+  end
+end
+

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: concurrent_downloader
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Oliver Clarke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Concurrently download http requests in a synchronous environment.
+email:
+- ollie@clarketus.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- concurrent_downloader.gemspec
+- lib/concurrent_downloader.rb
+- lib/concurrent_downloader/processor.rb
+- lib/concurrent_downloader/response.rb
+- lib/concurrent_downloader/version.rb
+- spec/lib/concurrent_downloader_spec.rb
+- spec/spec_helper.rb
+- spec/support/mock_server_helper.rb
+homepage: https://github.com/clarketus/concurrent_downloader
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: A basic em-http-request wrapper that slots into a normal ruby script.
+test_files:
+- spec/lib/concurrent_downloader_spec.rb
+- spec/spec_helper.rb
+- spec/support/mock_server_helper.rb