wire 0.1.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1,5 @@
+--color
+-fs
+-Ilib
+-Ispec
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in wire.gemspec
+gemspec

data/README.markdown

@@ -0,0 +1,98 @@
+# Wire
+
+Run a strict amount of threads during a time interval, primarily used for [web scraping](http://en.wikipedia.org/wiki/Web_scraping).
+
+## How to use
+
+### Example 1 - Basic
+
+Start 100 threads, only run 10 at the same time, with a 3 second delay between each new thread, except the first 10.
+
+    100.times do
+      Wire.new(max: 10, wait: 3) do
+        # Do stuff
+      end
+    end
+
+### Example 2 - Timer
+
+    11.times do
+      Wire.new(max: 10, wait: 1) do
+        sleep 0.1
+      end
+    end
+
+Time to run: ~ *1.2 seconds*.
+
+This is how it works.
+
+- 11 threads is created, done at time 0
+- Running 10 threads, done at time 0.1
+- Wait 1 second, done at time 1.1
+- Start the 11th thread, done at time 1.2
+    
+### Example 3 - Pass arguments
+
+    Wire.new(max: 10, wait: 1, vars: ["A", "B"]) do |first, last|
+      puts first # => "A"
+      puts last # => "B"
+    end
+
+    100.times do |n|
+      Wire.new(max: 10, wait: 1, vars: [n]) do |counter|
+        puts counter
+      end
+    end
+    
+    # => 1 2 3 4 5 ...
+
+### Example 4 - Scraping
+
+This project was originally build to solve the request limit problem when using [Spotify´s Meta API](http://developer.spotify.com/en/metadata-api/overview/).
+
+> In order to make the Metadata API snappy and open for everyone to use, rate limiting rules apply. If you make too many requests too fast, you’ll start getting 403 Forbidden responses. When rate limiting has kicked in, you’ll have to wait 10 seconds before making more requests. The rate limit is currently 10 request per second per ip. This may change.
+
+We wanted to make as many request as possible without being banned due to the rate limit.
+
+    require "rest-client"
+    require "wire"
+    require "uri"
+
+    a_very_large_list_of_songs = ["Sweet Home Alabama", ...]
+    
+    a_very_large_list_of_songs.each do |s|
+      Wire.new(max: 10, wait: 1, vars: [s]) do |song|
+        data = RestClient.get "http://ws.spotify.com/search/1/track.json?q=#{URI.encode(song)}"
+        # Do something with the data
+      end
+    end
+
+### Tip
+
+Don't forget to join your threads using `Thread#join`.
+ 
+    list = []
+    10.times do |n|
+      list << Thread.new do
+        # Do stuff
+      end
+    end
+    list.map(&:join)
+    
+Read more about [#join](http://corelib.rubyonrails.org/classes/Thread.html#M001145) here.
+
+## Arguments to pass
+
+Ingoing arguments to `new`.
+
+- **max** (Integer) The maximum amount of threads to run a the same time. The value 10 will be used if `max` is nil or zero.
+- **wait** (Integer) The time to wait before starting a new thread.
+- **vars** (Array) A list of arguments to the block.
+    
+## How do install
+
+    [sudo] gem install wire
+    
+## Requirements
+
+Wire is tested on OS X 10.6.7 using Ruby 1.9.2.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/wire.rb

@@ -0,0 +1,65 @@
+require "thread"
+require "monitor"
+
+class Wire < Thread  
+  def self.counter
+    @counter ||= Counter.new
+  end
+  
+  def initialize(args, &block)
+    args.keys.each { |name| instance_variable_set "@" + name.to_s, args[name] }
+    
+    if @max.to_i <= 0 or @wait.nil?
+      warn "Both max and wait needs to be passed, where max > 0. Using default values"
+      @max = 10 if @max.to_i <= 0
+      @wait ||= 1
+    end
+    
+    @block   = block
+    @counter = Wire.counter
+    
+    @counter.synchronize do
+      @counter.cond.wait_until { @counter.i < @max }
+      @counter.inc
+    end
+    
+    if @counter.last and (t = Time.now.to_f - @counter.last) < @wait
+      sleep (@wait - t)
+    end
+    
+    super { runner }
+  end
+  
+  def runner
+    @block.call(*@vars)
+  rescue => error
+    raise error
+  ensure
+    @counter.synchronize do
+      if @max == @counter.i or @counter.last
+        @counter.last = Time.now.to_f
+      end
+      @counter.dec
+      @counter.cond.signal
+    end
+  end
+end
+
+class Counter
+  attr_reader :i, :cond
+  attr_accessor :last
+  
+  def initialize
+    extend(MonitorMixin)
+    @i = 0
+    @cond = new_cond
+  end
+  
+  def inc
+    @i += 1
+  end
+
+  def dec
+    @i -= 1
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require "rspec"
+require "wire"
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+end

data/spec/wire_spec.rb

@@ -0,0 +1,103 @@
+require "spec_helper"
+
+def runner(times, options)
+  list = []
+  times.times do |n|
+    list << Wire.new(options) do |var|
+      yield
+    end
+  end
+  list.map(&:join)
+end
+
+def time
+  Time.now.to_f
+end
+
+describe Wire do
+  before(:each) do
+    counter = Counter.new
+    Wire.should_receive(:counter).any_number_of_times.and_return(counter)
+  end
+  
+  context "should be able to do run within the time limit" do
+    it "< max threads" do
+      start = time
+      runner(10, {max: 10, wait: 1}) do
+        sleep 0.1
+      end
+  
+      (time - start).should < 1.15
+    end
+  
+    it "> max threads" do
+      start = time
+      runner(11, {max: 10, wait: 1}) do
+        sleep 0.1
+      end
+  
+      (time - start).should > 1.2
+    end
+  end
+  
+  it "should run using one thread" do
+    start = time
+    runner(1, {max: 1, wait: 1}) do
+      sleep 0.1
+    end
+  
+    (time - start).should < 0.2
+  end
+  
+  it "should run using one thread, using a high max value" do
+    start = time
+    runner(1, {max: 100, wait: 1}) do
+      sleep 0.1
+    end
+
+    (time - start).should < 0.2
+  end
+  
+  it "it should not wait" do
+    start = time
+    runner(11, {max: 10, wait: 0}) do
+      sleep 0.1
+    end
+  
+    (time - start).should < 0.25
+  end
+  
+  context "error" do
+    it "should use the default values if wrong arguments is being passed" do
+      w = Wire.new(max: 0) {}.join
+      w.instance_eval do
+        @max.should == 10
+        @wait.should == 1
+      end
+    end
+    
+    it "should use the default values if nothing is being passed" do
+      w = Wire.new({}) {}.join
+      w.instance_eval do
+        @max.should == 10
+        @wait.should == 1
+      end
+    end
+    
+    it "should use the default values if wrong arguments is being passed" do
+      w = Wire.new(wait: 5) {}.join
+      w.instance_eval do
+        @max.should == 10
+        @wait.should == 5
+      end
+    end
+    
+    it "should be possible to raise an error" do
+      lambda do
+        Wire.new(wait: 5, max: 1) do
+          raise StandardError.new
+        end.join
+      end.should raise_error(StandardError)
+    end
+  end
+end

data/wire.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "wire"
+  s.version     = "0.1.0"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Linus Oleander"]
+  s.email       = ["linus@oleander.nu"]
+  s.homepage    = "https://github.com/Oleander/Wire"
+  s.summary     = %q{Run a strict amount of threads during a time interval}
+  s.description = %q{Run a strict amount of threads during a time interval, primarily used for web scraping.}
+
+  s.rubyforge_project = "wire"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.required_ruby_version = "~> 1.9.0"
+  s.add_development_dependency("rspec", "~> 2.5.0")
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: wire
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Linus Oleander
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-26 23:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.5.0
+  type: :development
+  version_requirements: *id001
+description: Run a strict amount of threads during a time interval, primarily used for web scraping.
+email: 
+- linus@oleander.nu
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- README.markdown
+- Rakefile
+- lib/wire.rb
+- spec/spec_helper.rb
+- spec/wire_spec.rb
+- wire.gemspec
+has_rdoc: true
+homepage: https://github.com/Oleander/Wire
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ~>
+    - !ruby/object:Gem::Version 
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: wire
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: Run a strict amount of threads during a time interval
+test_files: 
+- spec/spec_helper.rb
+- spec/wire_spec.rb