curb_threadpool 0.5.2 → 0.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dea25e638cb62f03dc2279dfe54ffdccc953c0f5
+  data.tar.gz: 830d96224e24f871ba618e9dfb2aae359bd66827
+SHA512:
+  metadata.gz: a4dca1c08fe2cfbab46425d8a6da4107af61f34b841ac2868d5a806954ce62398eb272352e68a40713a899c3d93989ac5511fa20311b97bf0f21651f87600b28
+  data.tar.gz: 371902e8e65bcb3c2f5b37040e7d4b366872c8e2bef25e7d7fbb35674f2dc9f22353ed6a9b3788c32e9d76f38ab8f21ea7d9d55a8818ac1e92579cdde8d982b4

data/Gemfile

@@ -1,12 +1,21 @@
 source "http://rubygems.org"
 
+gem "curb"
+
 group :development do
   gem "yard"
-  gem "redcarpet"
-  gem "shoulda", ">= 0"
-  gem "rdoc", "~> 3.12"
-  gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.8.3"
-  gem "rcov", ">= 0", :platforms => :mri_18
-  gem "simplecov", :platforms => :mri_19
+  gem "bundler"
+  gem "jeweler"
+
+  gem "minitest", "<5.0"
+  gem "webmock"
+
+  gem "rcov", :platforms => :mri_18
+  gem "simplecov", :platforms => [:mri_19, :mri_20]
+
+  # test utils
+  gem "test_guard", :git => "https://github.com/chetan/test_guard.git"
+  gem 'rb-inotify', :require => false
+  gem 'rb-fsevent', :require => false
+  gem 'rb-fchange', :require => false
 end

data/VERSION

@@ -1 +1 @@
-0.5.2
+0.6.0

data/curb_threadpool.gemspec

@@ -4,14 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{curb_threadpool}
-  s.version = "0.5.2"
+  s.name = "curb_threadpool"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Chetan Sarva"]
-  s.date = %q{2012-05-08}
-  s.description = %q{A multi-threaded worker pool for Curb}
-  s.email = %q{chetan@pixelcop.net}
+  s.date = "2013-07-31"
+  s.description = "A multi-threaded worker pool for Curb"
+  s.email = "chetan@pixelcop.net"
   s.extra_rdoc_files = [
     "LICENSE",
     "README.md"
@@ -27,43 +27,55 @@ Gem::Specification.new do |s|
     "test/helper.rb",
     "test/test_curb_threadpool.rb"
   ]
-  s.homepage = %q{http://github.com/chetan/curb_threadpool}
+  s.homepage = "http://github.com/chetan/curb_threadpool"
   s.licenses = ["BSD"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.5.0}
-  s.summary = %q{A multi-threaded worker pool for Curb}
+  s.rubygems_version = "2.0.3"
+  s.summary = "A multi-threaded worker pool for Curb"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<curb>, [">= 0"])
       s.add_development_dependency(%q<yard>, [">= 0"])
-      s.add_development_dependency(%q<redcarpet>, [">= 0"])
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<minitest>, ["< 5.0"])
+      s.add_development_dependency(%q<webmock>, [">= 0"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<test_guard>, [">= 0"])
+      s.add_development_dependency(%q<rb-inotify>, [">= 0"])
+      s.add_development_dependency(%q<rb-fsevent>, [">= 0"])
+      s.add_development_dependency(%q<rb-fchange>, [">= 0"])
     else
+      s.add_dependency(%q<curb>, [">= 0"])
       s.add_dependency(%q<yard>, [">= 0"])
-      s.add_dependency(%q<redcarpet>, [">= 0"])
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<rdoc>, ["~> 3.12"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<minitest>, ["< 5.0"])
+      s.add_dependency(%q<webmock>, [">= 0"])
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<test_guard>, [">= 0"])
+      s.add_dependency(%q<rb-inotify>, [">= 0"])
+      s.add_dependency(%q<rb-fsevent>, [">= 0"])
+      s.add_dependency(%q<rb-fchange>, [">= 0"])
     end
   else
+    s.add_dependency(%q<curb>, [">= 0"])
     s.add_dependency(%q<yard>, [">= 0"])
-    s.add_dependency(%q<redcarpet>, [">= 0"])
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<rdoc>, ["~> 3.12"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<minitest>, ["< 5.0"])
+    s.add_dependency(%q<webmock>, [">= 0"])
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<test_guard>, [">= 0"])
+    s.add_dependency(%q<rb-inotify>, [">= 0"])
+    s.add_dependency(%q<rb-fsevent>, [">= 0"])
+    s.add_dependency(%q<rb-fchange>, [">= 0"])
   end
 end
 

data/lib/curb_threadpool.rb

@@ -1,22 +1,33 @@
 
+require "curb"
+
 # A multi-threaded worker pool for Curb
-class Curl::ThreadPool
+module Curl
+class ThreadPool
+
+  class Request
+    # methods supported: :delete, :get, :head, :post, :put
+    attr_accessor :key, :uri, :body, :method
+    def initialize(key, uri, method=:get, body=nil)
+      @key    = key
+      @uri    = uri
+      @method = method
+      @body   = body
+    end
+  end
 
-  attr_reader :results
+  attr_reader :reqs, :results
 
   def initialize(size=4)
     @size = size
     reset()
   end
 
-  # Add a URL to be fetched
-  #
-  # @param [Object] key  to use for request
-  # @param [String] url  URL to fetch
-  def []=(key, url)
-    @reqs[key] = url
+  # Shorthand for adding requests
+  def add(req)
+    @reqs << req
   end
-  alias :add :[]=
+  alias_method :<<, :add
 
   # Wait for all threads to complete
   def join
@@ -26,82 +37,132 @@ class Curl::ThreadPool
 
   # Close all active Curl connections
   def close
-    @clients.each { |c| c.reset(); c.close() } if @clients
+    if @clients then
+      @clients.each { |c| c.reset(); c.close() }
+    end
   end
 
   # Reset the ThreadPool
   def reset
     close()
-    @reqs = {}
+    @reqs = []
     @results = {}
     @clients = []
     @threads = []
     @size.times{ @clients << Curl::Easy.new }
   end
 
-  # Utility method for retrieving a list of URLs
+  # Send multiple get requests
   #
   # @param [Array<String>] urls   list of URLs
+  #
   # @return [Array] array of response bodies
   def get(urls)
     if urls.nil? or urls.empty? then
-      return {}
+      return []
     end
 
     urls = [urls] if not urls.kind_of? Array
     urls.each_with_index do |url, i|
-      @reqs[i] = url.to_s
+      @reqs << Request.new(i, url.to_s)
     end
 
-    results = perform()
+    return collate_results(perform())
+  end
 
-    ret = []
-    (0..results.size-1).each do |i|
-      ret << results[i]
+  # Send multiple post requests
+  #
+  # @param [Array] reqs
+  def post(reqs)
+    if reqs.nil? or reqs.empty? then
+      return []
     end
 
-    return ret
+    if not reqs.first.kind_of? Array then
+      reqs = [ reqs ]
+    end
+
+    reqs.each_with_index do |r, i|
+      if r.kind_of? Request then
+        @reqs << r
+      elsif r.kind_of? Array then
+        @reqs << Request.new(i, r.shift, :post, r.shift)
+      end
+    end
+
+    return collate_results(perform())
   end
 
   # Execute requests. By default, will block until complete and return results.
   #
-  # @param [Boolean] no_block   If true, will not wait for requests to finish.
+  # @param [Boolean] async      If true, will not wait for requests to finish.
   #                             (Default=false)
   #
   # @param [Block] block        If passed, responses will be passed into the callback
   #                             instead of being returned directly
   #
-  # @return [Hash<Key, String>] Hash of responses, if no block given
-  def perform(no_block=false, &block)
+  # @yield [Request, String]    Passes to the block the request and the response body
+  #
+  # @return [Hash<Key, String>] Hash of responses, if no block given. Returns true otherwise
+  def perform(async=false, &block)
 
-    @results = {} if not @results.empty?
+    @results = {}
 
     @clients.each do |client|
       @threads << Thread.new do
 
         loop do
           break if @reqs.empty?
-          (key, url) = @reqs.shift
-          client.url = url
-          client.http_get
+          req = @reqs.shift
+          client.url = req.uri
+
+          args = ["http_#{req.method}"]
+          if [:put, :post].include? req.method
+            # add body to args for these methods
+            if req.body then
+              if req.body.kind_of? Array then
+                args += req.body
+              else
+                args << req.body
+              end
+            else
+              args << ""
+            end
+          end
+
+          client.send(*args)
           if block then
-            yield(key, client.body_str)
+            yield(req, client.body_str)
           else
-            @results[key] = client.body_str
+            @results[req.key] = client.body_str
           end
         end
 
       end
     end
 
-    return {} if no_block
+    if async then
+      # don't wait for threads to join, just return
+      return true
+    end
 
     join()
     return true if block
 
-    ret = @results
-    @results = {}
+    return @results
+  end
+
+
+  private
+
+  # Create ordered array from hash of results
+  def collate_results(results)
+    ret = []
+    results.size.times do |i|
+      ret << results[i]
+    end
     return ret
   end
 
-end
+end # ThreadPool
+end # Curl

data/test/helper.rb

@@ -7,12 +7,16 @@ rescue Bundler::BundlerError => e
   $stderr.puts "Run `bundle install` to install missing gems"
   exit e.status_code
 end
-require 'test/unit'
-require 'shoulda'
+
+require 'test_guard'
+TestGuard.load_simplecov()
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'curb_threadpool'
+require 'webmock/minitest'
 
-class Test::Unit::TestCase
+class MiniTest::Unit::TestCase
 end
+
+MiniTest::Unit.autorun

data/test/test_curb_threadpool.rb

@@ -1,7 +1,90 @@
 require 'helper'
 
-class TestCurbThreadpool < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+class TestCurbThreadpool < MiniTest::Unit::TestCase
+
+  def setup
+    super
+    WebMock.reset!
+    @url = "http://www.google.com"
+    @pool = Curl::ThreadPool.new
+    @i = 0
+  end
+
+  def teardown
+    super
+    @pool.reset()
+  end
+
+  def test_get
+    stub(:get)
+    ret = @pool.get(@url)
+
+    assert ret
+    assert_equal 1, ret.size
+    assert_equal "res 1", ret.first
+  end
+
+  def test_get_array_param
+    stub(:get)
+    ret = @pool.get([@url])
+
+    assert ret
+    assert_equal 1, ret.size
+    assert_equal "res 1", ret.first
+  end
+
+  def test_get_multiple
+    stub(:get)
+    urls = [@url, @url, @url]
+    ret = @pool.get(urls)
+    assert ret
+    assert_equal 3, ret.size
+    ret.each_with_index do |r, idx|
+      assert_equal "res #{idx+1}", r
+    end
+  end
+
+  def test_get_nil
+    pool = Curl::ThreadPool.new
+    ret = pool.get([])
+    assert_kind_of Array, ret
+    assert_empty ret
+  end
+
+  def test_post
+    stub(:post)
+    pool = Curl::ThreadPool.new
+    ret = pool.post([@url, ""])
+
+    assert ret
+    assert_equal 1, ret.size
+    assert_equal "res 1", ret.first
+  end
+
+  def test_post_multiple
+    stub(:post)
+    pool = Curl::ThreadPool.new
+    reqs = [
+      [@url, "foobar"],
+      [@url, "blah"]
+    ]
+    ret = pool.post(reqs)
+
+    assert ret
+    assert_equal 2, ret.size
+    assert_equal "res 1", ret.first
+    assert_equal "res 2", ret.last
+  end
+
+
+  private
+
+  def stub(method)
+    stub = stub_request(method, "http://www.google.com/").
+      to_return {
+        @i += 1
+        { :status => 200, :body => "res #{@i}" }
+      }
   end
+
 end

metadata

@@ -1,150 +1,191 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: curb_threadpool
-version: !ruby/object:Gem::Version 
-  hash: 15
-  prerelease: 
-  segments: 
-  - 0
-  - 5
-  - 2
-  version: 0.5.2
+version: !ruby/object:Gem::Version
+  version: 0.6.0
 platform: ruby
-authors: 
+authors:
 - Chetan Sarva
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-05-08 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: yard
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id001
-- !ruby/object:Gem::Dependency 
-  name: redcarpet
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id002
-- !ruby/object:Gem::Dependency 
-  name: shoulda
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '5.0'
   type: :development
-  requirement: *id003
-- !ruby/object:Gem::Dependency 
-  name: rdoc
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 3
-        - 12
-        version: "3.12"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id004
-- !ruby/object:Gem::Dependency 
-  name: bundler
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id005
-- !ruby/object:Gem::Dependency 
-  name: jeweler
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 49
-        segments: 
-        - 1
-        - 8
-        - 3
-        version: 1.8.3
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id006
-- !ruby/object:Gem::Dependency 
-  name: rcov
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test_guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-inotify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id007
-- !ruby/object:Gem::Dependency 
-  name: simplecov
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id008
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-fchange
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A multi-threaded worker pool for Curb
 email: chetan@pixelcop.net
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE
 - README.md
-files: 
+files:
 - Gemfile
 - LICENSE
 - README.md
@@ -154,39 +195,28 @@ files:
 - lib/curb_threadpool.rb
 - test/helper.rb
 - test/test_curb_threadpool.rb
-has_rdoc: true
 homepage: http://github.com/chetan/curb_threadpool
-licenses: 
+licenses:
 - BSD
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.0
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A multi-threaded worker pool for Curb
 test_files: []
-