curb_threadpool 0.5.0

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+group :development do
+  gem "yard"
+  gem "redcarpet"
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.8.3"
+  gem "rcov", ">= 0", :platforms => :mri_18
+  gem "simplecov", :platforms => :mri_19
+end

data/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2012, Chetan Sarva
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,41 @@
+# curb_threadpool
+
+A multi-threaded worker pool for Curb.
+
+## Usage
+
+    ct = Curl::ThreadPool.new
+    reqs = []
+    10000.times { |i| ct[i] = "http://localhost/test" }
+    responses = ct.exec() # => contains keyed response hash
+
+or
+
+    ct = Curl::ThreadPool.new
+    reqs = []
+    10000.times { reqs << "http://localhost/test" }
+    ct.get(reqs) # => contains responses in same order as reqs
+
+In order to re-use connections opened by the pool, hang on to it:
+
+    Thread.current[:my_pool] ||= Curl::ThreadPool.new
+    Thread.current[:my_pool].get("http://localhost/test")
+
+This way, if you frequently make requests to the same host (e.g., an API
+service) then the connections will be kept open as long as possible and
+re-used, speeding up your response time. Curl will automatically
+reconnected when necessary.
+
+## Contributing to curb_threadpool
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2012 Chetan Sarva. See LICENSE for further details.

data/Rakefile

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "curb_threadpool"
+  gem.homepage = "http://github.com/chetan/curb_threadpool"
+  gem.license = "BSD"
+  gem.summary = %Q{A multi-threaded worker pool for Curb}
+  gem.description = %Q{A multi-threaded worker pool for Curb}
+  gem.email = "chetan@pixelcop.net"
+  gem.authors = ["Chetan Sarva"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+if Object.const_defined? :Rcov
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+    test.rcov_opts << '--exclude "gems/*"'
+  end
+end
+
+task :default => :test
+
+require 'yard'
+YARD::Rake::YardocTask.new do |t|
+  t.options = [ "--files", "README.md,LICENSE" ]
+end

data/VERSION

@@ -0,0 +1 @@
+0.5.0

data/curb_threadpool.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{curb_threadpool}
+  s.version = "0.5.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Chetan Sarva"]
+  s.date = %q{2012-05-08}
+  s.description = %q{A multi-threaded worker pool for Curb}
+  s.email = %q{chetan@pixelcop.net}
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "curb_threadpool.gemspec",
+    "lib/curb_threadpool.rb",
+    "test/helper.rb",
+    "test/test_curb_threadpool.rb"
+  ]
+  s.homepage = %q{http://github.com/chetan/curb_threadpool}
+  s.licenses = ["BSD"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.5.0}
+  s.summary = %q{A multi-threaded worker pool for Curb}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<yard>, [">= 0"])
+      s.add_development_dependency(%q<redcarpet>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+    else
+      s.add_dependency(%q<yard>, [">= 0"])
+      s.add_dependency(%q<redcarpet>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<yard>, [">= 0"])
+    s.add_dependency(%q<redcarpet>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+  end
+end
+

data/lib/curb_threadpool.rb

@@ -0,0 +1,104 @@
+
+# A multi-threaded worker pool for Curb
+class Curl::ThreadPool
+
+  attr_reader :results
+
+  def initialize(size=4)
+    @size = size
+    reset()
+  end
+
+  # Add a URL to be fetched
+  #
+  # @param [Object] key  to use for request
+  # @param [String] url  URL to fetch
+  def []=(key, url)
+    @reqs[key] = url
+  end
+  alias :add :[]=
+
+  # Wait for all threads to complete
+  def join
+    @threads.each { |t| t.join }
+    @threads.clear
+  end
+
+  # Close all active Curl connections
+  def close
+    @clients.each { |c| c.reset(); c.close() } if @clients
+  end
+
+  # Reset the ThreadPool
+  def reset
+    close()
+    @reqs = {}
+    @results = {}
+    @clients = []
+    @threads = []
+    @size.times{ @clients << Curl::Easy.new }
+  end
+
+  # Utility method for retrieving a list of URLs
+  #
+  # @param [Array<String>] urls   list of URLs
+  # @return [Array] array of response bodies
+  def get(urls)
+    if urls.nil? or urls.empty? then
+      return {}
+    end
+
+    urls = [urls] if not urls.kind_of? Array
+    urls.each_with_index do |url, i|
+      @reqs[i] = url.to_s
+    end
+
+    results = exec()
+
+    ret = []
+    (0..results.size-1).each do |i|
+      ret << results[i]
+    end
+
+    return ret
+  end
+
+  # Execute requests. By default, will block until complete and return results.
+  #
+  # @param [Boolean] no_block   If true, will not wait for requests to finish.
+  #                             (Default=false)
+  #
+  # @param [Block] block        If passed, responses will be passed into the callback
+  #                             instead of being returned directly
+  #
+  # @return [Hash<Key, String>] Hash of responses, if no block given
+  def exec(no_block=false, &block)
+    @clients.each do |client|
+      @threads << Thread.new do
+
+        loop do
+          break if @reqs.empty?
+          (key, url) = @reqs.shift
+          client.url = url
+          client.http_get
+          if block then
+            yield(key, client.body_str)
+          else
+            @results[key] = client.body_str
+          end
+        end
+
+      end
+    end
+
+    return {} if no_block
+
+    join()
+    return true if block
+
+    ret = @results
+    @results = {}
+    return ret
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'curb_threadpool'
+
+class Test::Unit::TestCase
+end

data/test/test_curb_threadpool.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestCurbThreadpool < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,192 @@
+--- !ruby/object:Gem::Specification 
+name: curb_threadpool
+version: !ruby/object:Gem::Version 
+  hash: 11
+  prerelease: 
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
+platform: ruby
+authors: 
+- Chetan Sarva
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-08 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  name: redcarpet
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id002
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id003
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 3
+        - 12
+        version: "3.12"
+  prerelease: false
+  type: :development
+  requirement: *id004
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  prerelease: false
+  type: :development
+  requirement: *id005
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 1
+        - 8
+        - 3
+        version: 1.8.3
+  prerelease: false
+  type: :development
+  requirement: *id006
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id007
+- !ruby/object:Gem::Dependency 
+  name: simplecov
+  version_requirements: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  type: :development
+  requirement: *id008
+description: A multi-threaded worker pool for Curb
+email: chetan@pixelcop.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- curb_threadpool.gemspec
+- lib/curb_threadpool.rb
+- test/helper.rb
+- test/test_curb_threadpool.rb
+has_rdoc: true
+homepage: http://github.com/chetan/curb_threadpool
+licenses: 
+- BSD
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: A multi-threaded worker pool for Curb
+test_files: []
+