taskbag 1.0.2 → 2.0.0

data/README.md

@@ -1,6 +1,8 @@
 # TaskBag
 
-TODO: Write a gem description
+A super simple implementation of the Bag-of-Tasks Paradigm [1].
+
+This gem was originally created to scrap websites and download content in a parallel fashion.
 
 ## Installation
 
@@ -18,7 +20,51 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### When to use it
+
+You have [2]:
+* A bunch of tasks to do
+* They are all independent
+* Similar, maybe just different input files
+
+In TaskBag you can actually have different tasks in the same bag.
+
+### When not to use it
+
+If you don't in the above.
+
+### How to use it
+
+Define your job class:
+
+```
+class DownloadVideoJob < Struct.new(:video_url)
+  def run
+  	`wget -c #{self.video_url}`
+  end
+end
+```
+
+create your bag and open it with a number of workers:
+
+```
+bag = TaskBag::Bag.new
+bag.open 4
+```
+
+add your tasks however you want (maybe from another worker):
+
+```
+bag.add DownloadVideoJob.new('http://www.sometube.com/video.mp4')
+```
+
+don't *ever* forget to close the bag, or your application might finish before all the jobs are run:
+
+```
+bag.close!
+```
+
+As said before, you can add more tasks as the result of a job, for example, if you're writting a page scrapper, you can initiate your taskbag with only one job to scrap the index page, which will create other jobs to scrap each of the pages and find more jobs. The number of workers will never change unless you close and open your bag again.
 
 ## Contributing
 
@@ -27,3 +73,8 @@ TODO: Write usage instructions here
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## References
+
+[1] http://www.cs.arizona.edu/~greg/mpdbook/glossary.html
+[2] http://www.eead.csic.es/compbio/material/programacion_rocks/pics/paral_tareas.pdf

data/lib/taskbag/bag.rb

@@ -1,39 +1,39 @@
+require 'thread'
+
 module TaskBag
   class Bag
-    def initialize(task_class)
-      @tasks = []
-      @task_class = task_class
-      @semaphore = Mutex.new
+    def initialize(jobs=Queue.new)
+      @closed = true
+      @jobs = jobs
+      @threads = []
     end
 
-    def open(nthreads)
+    def open(nworkers)
+      raise "Bag is already opened!" unless closed?
       @closed = false
-      bag = self
-      @threads = nthreads.times.map do |w|
-        Thread.new { @task_class.new(bag).start }
+      _self = self
+      @threads = nworkers.times.map do
+        Thread.new { Worker.start(_self) }
       end
     end
 
-    def add(object)
-      @semaphore.synchronize {
-        @tasks << object
-      }
-    end
-
-    def close
-      loop { break unless @tasks.any? }
+    def close!
+      raise 'Bag is already closed!' if closed?
+      loop { break if @jobs.empty? }
       @closed = true
       @threads.each{|t| t.join}
     end
 
+    def add(object)
+      @jobs.push object
+    end
+
     def closed?
       !!@closed
     end
 
     def next
-      @semaphore.synchronize {
-        @tasks.pop
-      }
+      @jobs.pop unless @jobs.empty?
     end
   end
 end

data/lib/taskbag/version.rb

@@ -1,3 +1,3 @@
 module TaskBag
-  VERSION = '1.0.2'
+  VERSION = '2.0.0'
 end

data/lib/taskbag/worker.rb

@@ -4,15 +4,17 @@ module TaskBag
       @bag = bag
     end
 
+    def self.start(bag)
+      Worker.new(bag).start
+    end
+
     def start
       until @bag.closed?
-        task = @bag.next
-        (sleep(1) and next) if task.nil?
+        job = @bag.next
+        (sleep(1) and next) if job.nil?
 
-        self.run(task)
+        job.run
       end
     end
-
-    def run(task); end
   end
 end

data/spec/bag_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+
+describe TaskBag::Bag do
+	let(:mocked_queue) { double('mocked queue', empty?: true) }
+	subject { TaskBag::Bag.new }
+
+	it { should be_closed }
+
+	describe 'opening a bag' do
+		it 'makes the bag open' do
+			subject.open 0
+			subject.should_not be_closed
+			subject.close!
+		end
+
+		it 'raises an exception if already open' do
+			subject.open 0
+			expect { subject.open(0) }.to raise_error
+		end
+
+		it 'creates the given number of workers in different threads' do
+			TaskBag::Worker.should_receive(:start)
+										 .with(subject)
+										 .exactly(3).times
+										 .and_return 1, 2, 3
+
+			subject.open(3).should have(3).items
+			subject.close!
+		end
+	end
+
+	describe 'closing a bag' do
+		subject { TaskBag::Bag.new(mocked_queue) }
+
+		it 'should be closed after' do
+			mocked_queue.should_receive(:empty?).and_return true
+			subject.open 0
+			subject.close!
+			subject.should be_closed
+		end
+
+		it 'raises an exception if already closed' do
+			expect { subject.close! }.to raise_error
+		end
+
+		it 'waits for all jobs to be passed to workers before closing' do
+			mocked_queue.should_receive(:empty?).and_return false, false, true
+			subject.open 0
+			subject.close!
+		end
+
+		context 'a full bag' do
+			subject { TaskBag::Bag.new }
+			it 'waits for every worker to finish before it closes' do
+				finished = 0
+				job = double('job')
+				job.stub :run do
+					sleep 0.5
+					finished += 1
+				end
+				subject.open 2
+				subject.add job
+				subject.add job
+				subject.add job
+				subject.close!
+				finished.should == 3
+			end
+		end
+	end
+
+	it 'execute the jobs in a FIFO fashion' do
+		subject.add 1
+		subject.add 2
+
+		subject.next.should be 1
+		subject.next.should be 2
+		subject.next.should be_nil
+	end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require_relative '../lib/taskbag'

data/spec/worker_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe TaskBag::Worker do
+	let(:bag) { double('Bag')}
+	subject { TaskBag::Worker.new(bag) }
+
+	it 'runs tasks until bag is closed' do
+		job1, job2 = double('job 1'), double('job 2')
+		job1.should_receive(:run).with no_args
+		job2.should_receive(:run).with no_args
+		bag.should_receive(:closed?).and_return false, false, true
+		bag.should_receive(:next).and_return job1, job2
+
+		subject.start
+	end
+
+	it 'worker keep asking for jobs every second' do
+		job = double('job')
+		job.should_receive(:run).with no_args
+		bag.should_receive(:closed?).and_return false, false, true
+		bag.should_receive(:next).and_return nil, job
+
+		subject.should_receive(:sleep).with(1).and_return true
+
+		subject.start
+	end
+end

data/taskbag.gemspec

@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
 end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: taskbag
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
+  prerelease: 
 platform: ruby
 authors:
 - Carlos Palhares
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-23 00:00:00.000000000 Z
+date: 2013-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,15 +30,33 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: A simplistic task of bags implementation for multithreaded scripts
@@ -53,29 +74,36 @@ files:
 - lib/taskbag/bag.rb
 - lib/taskbag/version.rb
 - lib/taskbag/worker.rb
+- spec/bag_spec.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb
 - taskbag.gemspec
 homepage: https://github.com/xjunior/taskbag
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: A simplistic task of bags implementation for multithreaded scripts
-test_files: []
+test_files:
+- spec/bag_spec.rb
+- spec/spec_helper.rb
+- spec/worker_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 6104422a4067a937d2139e05fa644e676d7e0933
-  data.tar.gz: a354e49b9fa3deec00738d544e343d63fff1fb40
-SHA512:
-  metadata.gz: 4f0e5e4b172a6eb3a863658227bc5b044307270e0b423f75c3cd7f17b96adde322048edd50adaea96d81fb5b52dfe2fb29d914056afb1c984b90108874ea2126
-  data.tar.gz: bef99d3b30f7fe53e48844b18a5d77110e8c4643a0d897e45a25c2b827b00f9428a750164b0f3424f04ae44cd56b0ba8ef26701cfee9cf926bd37c3852c9e510