proco 0.0.1

data/test/test_proco.rb

@@ -0,0 +1,197 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+require 'parallelize'
+require 'logger'
+
+describe Proco do
+  before do
+    @logger = Logger.new($stdout)
+  end
+
+  it "can be created with method chaining" do
+    proco = Proco.interval(1).threads(4).queues(4).queue_size(100).batch(true).logger(@logger).new
+    opts = proco.options
+
+    assert_equal 1,       opts[:interval]
+    assert_equal 4,       opts[:threads]
+    assert_equal 4,       opts[:queues]
+    assert_equal 100,     opts[:queue_size]
+    assert_equal true,    opts[:batch]
+    assert_equal @logger, opts[:logger]
+
+    proco = Proco.threads(8).interval(10).new(:interval => 20)
+    opts = proco.options
+    assert_equal 20, opts[:interval]
+    assert_equal 8,  opts[:threads]
+  end
+
+  describe "in default setting" do
+    before do
+      @proco = Proco.new
+    end
+
+    it "yields each item" do
+      @proco.start do |items|
+        assert_equal false, items.is_a?(Array)
+      end
+
+      @proco.submit! 1
+      @proco.submit 2
+
+      @proco.exit
+    end
+
+    it "acts as a FCFS queue" do
+      processed = []
+      @proco.start do |item|
+        processed << item
+      end
+
+      feed = []
+      100.times do |i|
+        feed << i
+        @proco.submit! i
+      end
+      @proco.exit
+
+      assert_equal feed, processed
+      assert_equal 100, processed.length
+    end
+  end
+
+  describe "in batch-mode with multiple queues" do
+    before do
+      @proco = Proco.queue_size(10).queues(2).threads(4).batch(true).logger(@logger).new
+      @mutex = Mutex.new
+    end
+
+    it "always yields Array" do
+      bool = true
+      @proco.start do |items|
+        assert items.is_a?(Array)
+      end
+      @proco.submit 1
+      @proco.submit 2
+      @proco.submit! 3
+
+      @proco.exit
+    end
+
+    it "handles synchronous requests" do
+      cnt = 0
+      @proco.start do |items|
+        @mutex.synchronize do
+          cnt += items.inject(:+)
+        end
+      end
+      1000.times { |i| @proco.submit i }
+      assert_equal 1000.times.inject(:+), cnt
+
+      @proco.exit
+    end
+
+    it "handles asynchronous requests" do
+      cnt = 0
+      @proco.start do |items|
+        @mutex.synchronize do
+          cnt += items.inject(:+)
+        end
+      end
+      1000.times { |i| @proco.submit! i }
+      @proco.exit
+      assert_equal 1000.times.inject(:+), cnt
+    end
+  end
+
+  describe "in batch mode" do
+    it "yields batch_size items if set" do
+      proco = Proco.interval(0.1).batch_size(10).queue_size(1000).batch(true).new
+      cnt = 0
+      proco.start do |items|
+        # FIXME: naive
+        assert_equal 10, items.length
+        cnt += 1
+      end
+
+      100.times do |i|
+        proco.submit! i
+      end
+      proco.exit
+      assert_equal 10, cnt
+    end
+
+    it "works as follows" do
+      proco = Proco.interval(0.1).threads(4).queues(8).queue_size(10).batch(true).new
+
+      assert_equal false, proco.running?
+
+      # No you can't submit an item yet
+      assert_raises(RuntimeError) { proco.submit :not_yet }
+      assert_raises(RuntimeError) { proco.exit }
+      assert_raises(RuntimeError) { proco.kill }
+
+      assert_raises(ArgumentError) {
+        proco.start
+      }
+
+      proco.start do |batch|
+        # Batch-process items every 0.1 seconds
+        # ...
+        # puts "#{Thread.current}: #{batch}"
+        batch.length
+      end
+
+      assert_equal true, proco.running?
+
+      # Synchronous submit
+      50.times do |i|
+        result = proco.submit i
+        assert_instance_of Fixnum, result
+      end
+
+      # Asynchronous submit
+      futures = 50.times.map { |i|
+        proco.submit! i
+      }
+      futures.each do |future|
+        assert_instance_of Proco::Future, future
+      end
+
+      # Wait until the batch containing the item is processed
+      assert_equal 50, futures.uniq.map { |f| f.get }.inject(:+)
+
+      proco.exit
+      assert_equal false, proco.running?
+    end
+
+  end
+
+  it "can be killed without waiting" do
+    proco = Proco.batch(true).new
+    cnt = 0
+    proco.start do |items|
+      sleep 1
+      cnt += items.length
+    end
+    100.times do |i|
+      proco.submit! i
+    end
+    proco.exit
+    assert_equal 100, cnt
+
+    cnt = 0
+    proco.start do |items|
+      sleep 1
+      cnt += items.length
+    end
+    100.times do |i|
+      proco.submit! i
+    end
+    proco.kill
+    assert_equal 0, cnt
+  end
+end
+

data/test/test_queue.rb

@@ -0,0 +1,105 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+require 'lps'
+
+class TestQueue < MiniTest::Unit::TestCase
+  def test_single_queue
+    q = Proco::Queue::SingleQueue.new 100
+    f = q.push 1
+    assert_instance_of Proco::Future, f
+    q.push 2
+
+    f2, i = q.take
+    assert_equal f, f2
+    assert_equal 1, i
+
+    f3, i = q.take
+    assert_equal 2, i
+  end
+
+  def test_batch_queue
+    q = Proco::Queue::BatchQueue.new 100, 10
+
+    futures = 10.times.map { |i| q.push i }
+    assert_equal 1, futures.uniq.length
+
+    future = futures.first
+    future2 = q.push :next
+    assert future != future2
+
+    f, items = q.take
+    assert_equal future, f
+    assert_equal 10.times.to_a, items
+
+    f, items = q.take
+    assert_equal future2, f
+    assert_equal [:next], items
+
+    f = q.push :next
+    assert future2 != f
+  end
+
+  def test_multi_queue
+    q = Proco::Queue::MultiQueue.new 100
+    f1 = q.push 1
+    f2 = q.push 2
+    f3 = q.push 3
+    assert_equal f1, f2
+    assert_equal f2, f3
+
+    f4, items = q.take
+    assert_equal f3, f4
+    assert_equal [1, 2, 3], items
+
+    f5 = q.push 4
+    assert ! f4 != f5
+
+    f6, items = q.take
+    assert_equal f5, f6
+    assert_equal [4], items
+  end
+
+  def test_multi_queue_complex
+    queue = Proco::Queue::MultiQueue.new(200)
+    futures = []
+    num_batches = 0
+    num_items = 0
+
+    t1 = Thread.new {
+      1000.times do |i|
+        futures << queue.push(i)
+      end
+      queue.invalidate
+    }
+
+    t2 = Thread.new {
+      future = items = nil
+      begin
+        LPS.freq(10).while {
+          future, items = queue.take
+          future
+        }.loop do
+          num_batches += 1
+          num_items   += items.length
+
+          future.update do
+            items.length
+          end
+        end
+      rescue Exception => e
+        puts e
+      end
+    }
+
+    t1.join
+    t2.join
+
+    assert_equal 1000, futures.length
+    assert_equal num_batches, futures.uniq.length
+    assert_equal 1000, num_items
+    assert_equal 1000, futures.uniq.map { |future| future.get }.inject(:+)
+  end
+end

data/test/test_worker.rb

@@ -0,0 +1,19 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+
+class TestWorker < MiniTest::Unit::TestCase
+  def test_worker
+    w = Proco::MT::Worker.new nil
+    cnt = 0
+    w.assign { sleep 0.1; cnt += 1 }
+    w.assign { sleep 0.1; cnt += 1 }
+    w.assign { sleep 0.1; cnt += 1 } # Async
+    assert_equal 1, cnt
+    sleep 0.2;
+    w.exit
+    assert_equal 3, cnt
+  end
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: proco
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Junegunn Choi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lps
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: option_initializer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallelize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A lightweight asynchronous task executor service designed for efficient
+  batch processing
+email:
+- junegunn.c@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark/comparison.rb
+- benchmark/submission.rb
+- lib/proco.rb
+- lib/proco/dispatcher.rb
+- lib/proco/future.rb
+- lib/proco/logger.rb
+- lib/proco/mt/base.rb
+- lib/proco/mt/pool.rb
+- lib/proco/mt/threaded.rb
+- lib/proco/mt/worker.rb
+- lib/proco/queue/base.rb
+- lib/proco/queue/batch_queue.rb
+- lib/proco/queue/multi_queue.rb
+- lib/proco/queue/single_queue.rb
+- lib/proco/version.rb
+- proco.gemspec
+- test/test_mt_base.rb
+- test/test_mt_threaded.rb
+- test/test_pool.rb
+- test/test_proco.rb
+- test/test_queue.rb
+- test/test_worker.rb
+homepage: https://github.com/junegunn/proco
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A lightweight asynchronous task executor service designed for efficient batch
+  processing
+test_files:
+- test/test_mt_base.rb
+- test/test_mt_threaded.rb
+- test/test_pool.rb
+- test/test_proco.rb
+- test/test_queue.rb
+- test/test_worker.rb