proco 0.0.1

data/benchmark/submission.rb

@@ -0,0 +1,150 @@
+#!/usr/bin/env ruby
+
+$VERBOSE = true
+require 'rubygems'
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'proco'
+require 'benchmark'
+require 'parallelize'
+require 'thread'
+require 'logger'
+
+logger = Logger.new($stdout)
+times = 1_000_000
+
+if celluloid = false
+  require 'celluloid'
+  class Cell
+    include Celluloid
+
+    def push
+      Proco::Future.new
+    end
+  end
+end
+
+result = Benchmark.bm(40) do |x|
+  x.report('celluloid') do
+    c = Cell.new
+    f = nil
+    times.times do |i|
+      print '.' if i % 1000 == 0
+      c.push!
+    end
+    c.future.push.value
+  end if celluloid
+
+  x.report('simple loop') do
+    times.times do |i|
+      Proco::Future.new
+    end
+  end
+
+  x.report('Default Proco') do
+    proco = Proco.new
+    proco.start do |item|
+      nil
+    end
+
+    times.times do |i|
+      print '.' if i % 1000 == 0
+      proco.submit! i
+    end
+    proco.exit
+  end
+
+  x.report('SizedQueue push and pop') do
+    q = SizedQueue.new 1000
+    times.times do |i|
+      q.push Proco::Future.new
+      q.pop
+    end
+  end
+
+  x.report('SizedQueue push and pop (threads)') do
+    q = SizedQueue.new 1000
+    t1 = Thread.new do
+      times.times do |i|
+        q.push Proco::Future.new
+      end
+      q.push nil
+    end
+    t2 = Thread.new do
+      while q.pop
+      end
+    end
+
+    t1.join
+    t2.join
+  end
+
+  x.report('Mutex synchronization') do
+    m = Mutex.new
+    a = []
+    times.times do |i|
+      m.synchronize do
+        a << Proco::Future.new
+      end
+      m.synchronize do
+        a.shift
+      end
+    end
+  end
+
+  x.report('Proco queue') do
+    q = Proco::Queue::SingleQueue.new 1000
+    times.times do |i|
+      q.push i
+      q.take
+    end
+  end
+
+  x.report('Proco queue (thread)') do
+    q = Proco::Queue::SingleQueue.new 1000
+    t1 = Thread.new do
+      times.times do |i|
+        print '.' if i % 1000 == 0
+        q.push i
+      end
+      q.invalidate
+    end
+
+    t2 = Thread.new do
+      while true
+        f, i = q.take
+        break unless f
+      end
+    end
+
+    t1.join
+    t2.join
+  end
+
+  [1, 4, 16].each do |queues|
+    [1, 2, 4].each do |threads|
+      x.report("q: #{queues}, t: #{threads}") do
+        proco = Proco.queues(queues).logger(logger).threads(threads).new
+        proco.start do |items|
+          nil
+        end
+        threads = 1
+        parallelize(threads) do
+          (times / threads).times do |i|
+            print '.' if i % 1000 == 0
+            proco.submit! i
+          end
+        end
+        proco.kill
+      end
+    end
+  end
+end
+
+data = Hash[ result.map { |r| [r.label, r.real] } ]
+mlen = data.keys.map(&:length).max
+mval = data.values.max
+width = 40
+data.each do |k, v|
+  puts k.ljust(mlen) + ' : ' + '*' * (width * (v / mval)).to_i
+end
+

data/lib/proco.rb

@@ -0,0 +1,112 @@
+require 'proco/version'
+require 'proco/logger'
+require 'proco/mt/base'
+require 'proco/mt/threaded'
+require 'proco/mt/worker'
+require 'proco/mt/pool'
+require 'proco/dispatcher'
+require 'proco/future'
+require 'proco/queue/base'
+require 'proco/queue/single_queue'
+require 'proco/queue/batch_queue'
+require 'proco/queue/multi_queue'
+require 'option_initializer'
+
+class Array
+  def sample
+    self[Kernel.rand(size)]
+  end unless method_defined? :sample
+end
+
+class Proco
+  include Proco::Logger
+  include OptionInitializer
+
+  option_initializer :interval, :threads, :queues, :queue_size, :batch, :batch_size, :logger
+  option_validator do |opt, val|
+    case opt
+    when :interval
+      raise ArgumentError, "interval must be a number" unless val.is_a?(Numeric)
+    when :threads, :queues, :queue_size, :batch_size
+      raise ArgumentError, "#{opt} must be a positive non-zero integer" unless val.is_a?(Fixnum) && val > 0
+    when :batch
+      raise ArgumentError, "batch must be a boolean value" unless [true, false].include?(val)
+    end
+  end
+
+  attr_reader :options
+
+  DEFAULT_OPTIONS = {
+    :interval   => 0,
+    :threads    => 1,
+    :queues     => 1,
+    :queue_size => 1000,
+    :batch      => false
+  }
+
+  def initialize options = {}, &processor
+    validate_options options
+    @options = DEFAULT_OPTIONS.merge(options)
+    @logger = @options[:logger]
+
+    @pool = nil
+    @running = false
+  end
+
+  # @return [Proco]
+  def start &block
+    raise ArgumentError, "Block not given" if block.nil?
+    @running = true
+    @pool = Proco::MT::Pool.new(options[:threads], @logger)
+    @dispatchers = options[:queues].times.map { |i|
+      Dispatcher.new(self, @pool, block)
+    }
+
+    self
+  end
+
+  # Synchronous submission
+  # @param [Object] items
+  # @return [Hash]
+  def submit item
+    check_running
+    submit!(item).get
+  end
+
+  # Asynchronous submission
+  # @param [Object] items
+  # @return [Proco::Future]
+  def submit! item
+    check_running
+    @dispatchers.sample.push(item)
+  end
+
+  # Stops Proco, returns results from remaining submissions in the queue.
+  # @return [nil]
+  def exit
+    check_running
+    @running = false
+    @dispatchers.each(&:exit)
+    @pool.exit
+  end
+
+  # @return [nil]
+  def kill
+    check_running
+    @running = false
+    @dispatchers.each(&:kill)
+    @pool.kill
+    nil
+  end
+
+  # @return [Boolean]
+  def running?
+    @running
+  end
+
+private
+  def check_running
+    raise RuntimeError, "Not running" unless running?
+  end
+end
+

data/lib/proco/dispatcher.rb

@@ -0,0 +1,59 @@
+require 'lps'
+
+class Proco
+# @private
+class Dispatcher
+  include Proco::MT::Threaded
+
+  def initialize proco, thread_pool, block
+    super()
+
+    @logger, interval, qs, batch, batch_size =
+      proco.options.values_at :logger, :interval, :queue_size, :batch, :batch_size
+    @queue = if batch && batch_size
+               Proco::Queue::BatchQueue.new(qs, batch_size)
+             elsif batch
+               Proco::Queue::MultiQueue.new(qs)
+             else
+               Proco::Queue::SingleQueue.new(qs)
+             end
+    @pool  = thread_pool
+    @block = block
+
+    spawn do
+      future = items = nil
+      LPS.interval(interval).while {
+        future, items = @queue.take
+        future # JRuby bug
+      }.loop do
+        inner_loop future, items
+      end
+    end
+  end
+
+  def push *items
+    @queue.push(*items)
+  end
+
+  def exit
+    @queue.invalidate
+    super
+  end
+
+private
+  def inner_loop future, items
+    @pool.assign do
+      future.update do
+        begin
+          @block.call items
+        rescue Exception => e
+          error e.to_s # TODO
+          raise
+        end
+      end
+    end
+
+    future
+  end
+end#Dispatcher
+end#Proco

data/lib/proco/future.rb

@@ -0,0 +1,40 @@
+class Proco
+class Future
+  include Proco::MT::Base
+
+  def get
+    do_when(proc { @state != :wait }) do
+      if @state == :ok
+        return @return
+      else
+        raise @return
+      end
+    end
+  end
+
+  def inspect
+    "Future=#{@state}"
+  end
+
+  # @private
+  def initialize
+    super()
+    @state = :wait
+    @return = nil
+  end
+
+  # @private
+  def update
+    begin
+      @return = yield
+      @state = :ok
+    rescue Exception => e
+      @return = e
+      @state = :fail
+    end
+
+    broadcast
+  end
+end
+end
+

data/lib/proco/logger.rb

@@ -0,0 +1,21 @@
+require 'forwardable'
+
+class Proco
+module Logger
+  extend Forwardable
+
+  class DummyLogger
+    [:info, :debug, :warn, :error].each do |m|
+      define_method m do |msg|
+      end
+    end
+  end
+
+  def logger
+    @logger ||= DummyLogger.new
+  end
+
+  def_delegators :logger, :info, :debug, :warn, :error
+end#Logger
+end#Proco
+

data/lib/proco/mt/base.rb

@@ -0,0 +1,65 @@
+class Proco
+module MT
+# @private
+module Base
+  def initialize
+    @mtx = Mutex.new
+    @cv  = ConditionVariable.new
+  end
+
+  def try_when cond, &block
+    return unless @mtx.try_lock
+
+    begin
+      block.call if cond.call
+    ensure
+      @cv.broadcast
+      @mtx.unlock
+    end
+  end
+
+  def do_when cond, &block
+    @mtx.lock
+    while !cond.call
+      @cv.wait @mtx
+    end
+    block.call
+  ensure
+    # A good discussion on the use of broadcast instead of signal
+    # http://stackoverflow.com/questions/37026/java-notify-vs-notifyall-all-over-again
+    @cv.broadcast
+    @mtx.unlock
+  end
+
+  def synchronize
+    @mtx.synchronize do
+      yield
+    end
+  end
+
+  def wait_until &cond
+    do_when(cond) {}
+  end
+
+  def signal &block
+    @mtx.synchronize do
+      begin
+        block.call if block
+      ensure
+        @cv.signal
+      end
+    end
+  end
+
+  def broadcast &block
+    @mtx.synchronize do
+      begin
+        block.call if block
+      ensure
+        @cv.broadcast
+      end
+    end
+  end
+end#Base
+end#MT
+end