proco 0.0.1

data/lib/proco/mt/pool.rb

@@ -0,0 +1,53 @@
+class Proco
+module MT
+# @private
+class Pool
+  include Proco::Logger
+
+  def initialize size, logger = nil
+    @logger = logger
+    @workers = size.times.map { |i|
+      Worker.new @logger
+    }
+    @num_workers = @workers.length
+    if @num_workers > 1
+      self.instance_eval do
+        alias assign assign_try
+      end
+    else
+      self.instance_eval do
+        alias assign assign_wait
+      end
+    end
+  end
+
+  def assign_wait &block
+    # Optimistic randomized assignment to avoid mutex contention
+    @workers.sample.assign(&block)
+  end
+
+  def assign_try &block
+    @num_workers.times do |i|
+      ret = @workers.sample.try_assign(&block)
+      return ret if ret
+    end
+    # phew. blocking assignment
+    # debug "Failed immediate thread allocation in the 1st round (#@num_workers)"
+    assign_wait(&block)
+  end
+
+  def exit
+    @workers.each(&:exit)
+  end
+
+  def kill
+    @workers.each(&:kill)
+  end
+
+  def counter
+    @workers.map(&:counter).inject(:+)
+  end
+end#Pool
+end#MT
+end#Proco
+

data/lib/proco/mt/threaded.rb

@@ -0,0 +1,52 @@
+class Proco
+module MT
+# @private
+module Threaded
+  include Proco::Logger
+  include Proco::MT::Base
+
+  def initialize
+    super
+    @running = false
+  end
+
+  def running?
+    @running
+  end
+
+  def exit
+    broadcast do
+      @running = false
+    end
+    @thread.join
+  end
+
+  def kill
+    @running = false
+    Thread.kill @thread if @thread
+  end
+
+  def spawn &block
+    @thread = Thread.new do
+      debug "#{Thread.current} started (#{self})"
+      broadcast do
+        @running = true
+      end
+
+      begin
+        block.call
+      rescue Exception => e
+        error "[#{Thread.current}] #{e}"
+        raise
+      ensure
+        debug "#{Thread.current} exited (#{self})"
+      end
+    end
+    wait_until { running? }
+
+    @thread
+  end
+end#Threaded
+end#MT
+end#Proco
+

data/lib/proco/mt/worker.rb

@@ -0,0 +1,81 @@
+class Proco
+module MT
+# @private
+class Worker
+  include Proco::MT::Threaded
+
+  attr_reader :counter
+
+  def initialize logger
+    super()
+
+    @logger  = logger
+    @block   = nil
+    @counter = 0
+
+    spawn do
+      work while running?
+    end
+  end
+
+  def work
+    # Not using do_when makes the code around the task block about 10% faster
+    @mtx.lock
+    while true
+      break if @block
+      return unless running?
+      @cv.wait @mtx
+    end
+    block = @block
+    @counter += 1
+    @block = nil
+
+    # Release lock here, so that a new task can be assigned during the execution
+    # 50 -> 30
+    @cv.broadcast
+    @mtx.unlock
+
+    # Work!
+    block.call
+  end
+
+  # Blocks when working
+  def assign &block
+    #do_when(Proc.new { return unless running?; @block.nil? }) do
+    #  @block = block
+    #end
+    @mtx.lock
+    while true
+      break unless @block
+      return unless running?
+      @cv.wait @mtx
+    end
+    @block = block
+  ensure
+    @cv.broadcast
+    @mtx.unlock
+  end
+
+  # Returns nil when working
+  def try_assign &block
+    #try_when(Proc.new { return unless running?; @block.nil? }) do
+    #  @block = block
+    #end
+    return unless @mtx.try_lock
+
+    begin
+      return if !running? || @block
+      @block = block
+    ensure
+      @cv.broadcast
+      @mtx.unlock
+    end
+  end
+
+  def exit
+    wait_until { @block.nil? }
+    super
+  end
+end#Worker
+end#MT
+end#Proco

data/lib/proco/queue/base.rb

@@ -0,0 +1,58 @@
+require 'thread'
+
+class Proco
+module Queue
+# @private
+class Base
+  include Proco::MT::Base
+
+  class Invalidated < Exception
+    def to_s
+      "Queue invalidated"
+    end
+  end
+
+  def initialize size
+    super()
+    @size   = size
+    @items  = []
+    @valid  = true
+  end
+
+  def invalidate
+    broadcast do
+      @valid = false
+    end
+  end
+
+  def push item
+    @mtx.lock
+    while true
+      raise Invalidated unless @valid
+      break if @items.length < @size
+      @cv.wait @mtx
+    end
+    push_impl item
+  ensure
+    @cv.broadcast
+    @mtx.unlock
+  end
+
+  def take
+    @mtx.lock
+    while true
+      empty = @items.empty?
+      break unless empty
+      return nil unless @valid
+      @cv.wait @mtx
+    end
+    take_impl
+  ensure
+    @cv.broadcast
+    @mtx.unlock
+  end
+end#Base
+end#Queue
+end
+
+

data/lib/proco/queue/batch_queue.rb

@@ -0,0 +1,30 @@
+require 'thread'
+
+class Proco
+module Queue
+# @private
+class BatchQueue < Proco::Queue::Base
+  def initialize size, batch_size
+    super size
+    @futures = []
+    @batch_size = batch_size
+  end
+
+  def push_impl item
+    @items << item
+    if @items.length % @batch_size == 1
+      @futures << Future.new
+    end
+    @futures.last
+  end
+
+  def take_impl
+    items  = @items[0, @batch_size]
+    @items = @items[@batch_size..-1] || []
+
+    [@futures.shift, items]
+  end
+end
+end
+end
+

data/lib/proco/queue/multi_queue.rb

@@ -0,0 +1,30 @@
+require 'thread'
+
+class Proco
+module Queue
+# @private
+class MultiQueue < Proco::Queue::Base
+  def initialize size
+    super
+    @future = Future.new
+  end
+
+  def push_impl item
+    @items << item
+    @future
+  end
+
+  def take_impl
+    items   = @items
+    ret     = [@future, items]
+
+    # Reset vars
+    @items  = []
+    @future = Future.new
+
+    ret
+  end
+end
+end
+end
+

data/lib/proco/queue/single_queue.rb

@@ -0,0 +1,23 @@
+require 'thread'
+
+class Proco
+module Queue
+# @private
+class SingleQueue < Proco::Queue::Base
+  def initialize size
+    super
+  end
+
+  def push_impl item
+    future = Future.new
+    @items << [future, item]
+    future
+  end
+
+  def take_impl
+    @items.shift
+  end
+end
+end
+end
+

data/lib/proco/version.rb

@@ -0,0 +1,3 @@
+class Proco
+  VERSION = "0.0.1"
+end

data/proco.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'proco/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "proco"
+  gem.version       = Proco::VERSION
+  gem.authors       = ["Junegunn Choi"]
+  gem.email         = ["junegunn.c@gmail.com"]
+  gem.description   = %q{A lightweight asynchronous task executor service designed for efficient batch processing}
+  gem.summary       = %q{A lightweight asynchronous task executor service designed for efficient batch processing}
+  gem.homepage      = "https://github.com/junegunn/proco"
+
+  gem.files         = `git ls-files`.split($/).reject { |f| f =~ %r[^viz/] }
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency 'lps', '~> 0.1.3'
+  gem.add_runtime_dependency 'option_initializer', '~> 1.2.0'
+  gem.add_development_dependency 'minitest'
+  gem.add_development_dependency 'parallelize'
+end

data/test/test_mt_base.rb

@@ -0,0 +1,40 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+
+class MTBaseSub
+  include Proco::MT::Base
+
+  def initialize
+    super
+  end
+
+  def ok dur
+    a = false
+    t = Thread.new { sleep dur; a = :ok; self.signal }
+    do_when(proc { a }) do
+      a
+    end.tap { t.join }
+  end
+
+  def not_ok
+    # Proc.new instead of proc (1.8 compat)
+    do_when(Proc.new { return :return }) do
+      :anything
+    end
+  end
+end
+
+class TestMTBase < MiniTest::Unit::TestCase
+  def test_mt_base
+    t = MTBaseSub.new
+    st = Time.now
+    assert_equal :ok, t.ok(0.1)
+    assert Time.now - st >= 0.1 # FIXME
+
+    assert_equal :return, t.not_ok
+  end
+end
+

data/test/test_mt_threaded.rb

@@ -0,0 +1,32 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+
+class MTThreadedSub
+  include Proco::MT::Threaded
+
+  def initialize
+    super
+  end
+end
+
+class TestMTThreaded < MiniTest::Unit::TestCase
+  def test_mt_base
+    t = MTThreadedSub.new
+    assert_equal false, t.running?
+    status = nil
+    t.spawn do
+      while t.running?
+        sleep 0.01
+      end
+      status = :done
+    end
+    assert_equal true, t.running?
+    t.exit
+    assert_equal :done, status
+    assert_equal false, t.running?
+  end
+end
+

data/test/test_pool.rb

@@ -0,0 +1,20 @@
+$VERBOSE = true
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) if $0 == __FILE__
+require 'rubygems'
+require 'minitest/autorun'
+require 'proco'
+
+class TestPool < MiniTest::Unit::TestCase
+  def test_pool
+    pool = Proco::MT::Pool.new(8)
+    cnt = 0
+    mtx = Mutex.new
+    1000.times do
+      pool.assign { mtx.synchronize { cnt += 3 } }
+    end
+    pool.exit
+    assert_equal 1000, pool.counter
+    assert_equal 3000, cnt
+  end
+end
+