concurrent-ruby 0.5.0 → 0.6.0.pre.1

data/lib/concurrent/per_thread_executor.rb

@@ -0,0 +1,23 @@
+module Concurrent
+
+  class PerThreadExecutor
+
+    def self.post(*args)
+      raise ArgumentError.new('no block given') unless block_given?
+      Thread.new(*args) do
+        Thread.current.abort_on_exception = false
+        yield(*args)
+      end
+      return true
+    end
+
+    def post(*args, &block)
+      return PerThreadExecutor.post(*args, &block)
+    end
+
+    def <<(block)
+      PerThreadExecutor.post(&block)
+      return self
+    end
+  end
+end

data/lib/concurrent/postable.rb

@@ -75,6 +75,7 @@ module Concurrent
       end
     end
 
+    # @deprecated +Actor+ is being replaced with a completely new framework prior to v1.0.0
     def forward(receiver, *message)
       raise ArgumentError.new('empty message') if message.empty?
       return false unless ready?
@@ -82,6 +83,7 @@ module Concurrent
       queue.length
     end
 
+    # @deprecated +Actor+ is being replaced with a completely new framework prior to v1.0.0
     def ready?
       if self.respond_to?(:running?) && ! running?
         false

data/lib/concurrent/processor_count.rb

@@ -0,0 +1,125 @@
+require 'rbconfig'
+
+module Concurrent
+
+  # Number of processors seen by the OS and used for process scheduling. For performance
+  # reasons the calculated value will be memoized on the first call.
+  #
+  # When running under JRuby the Java runtime call +java.lang.Runtime.getRuntime.availableProcessors+
+  # will be used. According to the Java documentation this "value may change
+  # during a particular invocation of the virtual machine... [applications]
+  # should therefore occasionally poll this property." Subsequently the result
+  # will NOT be memoized under JRuby.
+  #
+  # On Windows the Win32 API will be queried for the `NumberOfLogicalProcessors from Win32_Processor`.
+  # This will return the total number "logical processors for the current instance of the processor",
+  # which taked into account hyperthreading.
+  #
+  # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
+  # * BSD: /sbin/sysctl
+  # * Cygwin: /proc/cpuinfo
+  # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
+  # * HP-UX: /usr/sbin/ioscan
+  # * IRIX: /usr/sbin/sysconf
+  # * Linux: /proc/cpuinfo
+  # * Minix 3+: /proc/cpuinfo
+  # * Solaris: /usr/sbin/psrinfo
+  # * Tru64 UNIX: /usr/sbin/psrinfo
+  # * UnixWare: /usr/sbin/psrinfo
+  #
+  # @return [Integer] number of processors seen by the OS or Java runtime
+  #
+  # @see https://github.com/grosser/parallel/blob/4fc8b89d08c7091fe0419ca8fba1ec3ce5a8d185/lib/parallel.rb
+  #
+  # @see http://docs.oracle.com/javase/6/docs/api/java/lang/Runtime.html#availableProcessors()
+  # @see http://msdn.microsoft.com/en-us/library/aa394373(v=vs.85).aspx
+  def processor_count
+    if RUBY_PLATFORM == 'java'
+      java.lang.Runtime.getRuntime.availableProcessors
+    else
+      @@processor_count ||= begin
+                              os_name = RbConfig::CONFIG["target_os"]
+                              if os_name =~ /mingw|mswin/
+                                require 'win32ole'
+                                result = WIN32OLE.connect("winmgmts://").ExecQuery(
+                                  "select NumberOfLogicalProcessors from Win32_Processor")
+                                result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
+                              elsif File.readable?("/proc/cpuinfo")
+                                IO.read("/proc/cpuinfo").scan(/^processor/).size
+                              elsif File.executable?("/usr/bin/hwprefs")
+                                IO.popen("/usr/bin/hwprefs thread_count").read.to_i
+                              elsif File.executable?("/usr/sbin/psrinfo")
+                                IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
+                              elsif File.executable?("/usr/sbin/ioscan")
+                                IO.popen("/usr/sbin/ioscan -kC processor") do |out|
+                                  out.read.scan(/^.*processor/).size
+                                end
+                              elsif File.executable?("/usr/sbin/pmcycles")
+                                IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
+                              elsif File.executable?("/usr/sbin/lsdev")
+                                IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
+                              elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
+                                IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
+                              elsif File.executable?("/usr/sbin/sysctl")
+                                IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
+                              elsif File.executable?("/sbin/sysctl")
+                                IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
+                              else
+                                1
+                              end
+                            end
+    end
+  rescue
+    return 1
+  end
+  module_function :processor_count
+
+  # Number of physical processor cores on the current system. For performance reasons
+  # the calculated value will be memoized on the first call.
+  #
+  # On Windows the Win32 API will be queried for the `NumberOfCores from Win32_Processor`.
+  # This will return the total number "of cores for the current instance of the processor."
+  # On Unix-like operating systems either the `hwprefs` or `sysctl` utility will be called
+  # in a subshell and the returned value will be used. In the rare case where none of these
+  # methods work or an exception is raised the function will simply return 1.
+  #
+  # @return [Integer] number physical processor cores on the current system
+  #
+  # @see https://github.com/grosser/parallel/blob/4fc8b89d08c7091fe0419ca8fba1ec3ce5a8d185/lib/parallel.rb
+  #
+  # @see http://msdn.microsoft.com/en-us/library/aa394373(v=vs.85).aspx
+  # @see http://www.unix.com/man-page/osx/1/HWPREFS/
+  # @see http://linux.die.net/man/8/sysctl
+  def physical_processor_count
+    @@physical_processor_count ||= begin
+                                     ppc = case RbConfig::CONFIG["target_os"]
+                                           when /darwin1/
+                                             IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
+                                           when /linux/
+                                             cores = {}  # unique physical ID / core ID combinations
+                                             phy = 0
+                                             IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
+                                               if ln.start_with?("physical")
+                                                 phy = ln[/\d+/]
+                                               elsif ln.start_with?("core")
+                                                 cid = phy + ":" + ln[/\d+/]
+                                                 cores[cid] = true if not cores[cid]
+                                               end
+                                             end
+                                             cores.count
+                                           when /mswin|mingw/
+                                             require 'win32ole'
+                                             result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
+                                               "select NumberOfCores from Win32_Processor")
+                                             result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
+                                           else
+                                             processor_count
+                                           end
+                                     # fall back to logical count if physical info is invalid
+                                     ppc > 0 ? ppc : processor_count
+                                   end
+  rescue
+    return 1
+  end
+  module_function :physical_processor_count
+end

data/lib/concurrent/promise.rb

@@ -1,22 +1,43 @@
 require 'thread'
 
-require 'concurrent/global_thread_pool'
+require 'concurrent/configuration'
 require 'concurrent/obligation'
 
 module Concurrent
 
   class Promise
     include Obligation
-    include UsesGlobalThreadPool
-
+    include OptionsParser
+
+    # Initialize a new Promise with the provided options.
+    #
+    # @param [Object] initial the initial value
+    # @param [Hash] opts the options used to define the behavior at update and deref
+    #
+    # @option opts [Promise] :parent the parent +Promise+ when building a chain/tree
+    # @option opts [Proc] :on_fulfill fulfillment handler
+    # @option opts [Proc] :on_reject rejection handler
+    #
+    # @option opts [Boolean] :operation (false) when +true+ will execute the future on the global
+    #   operation pool (for long-running operations), when +false+ will execute the future on the
+    #   global task pool (for short-running tasks)
+    # @option opts [object] :executor when provided will run all operations on
+    #   this executor rather than the global thread pool (overrides :operation)
+    #
+    # @option opts [String] :dup_on_deref (false) call +#dup+ before returning the data
+    # @option opts [String] :freeze_on_deref (false) call +#freeze+ before returning the data
+    # @option opts [String] :copy_on_deref (nil) call the given +Proc+ passing the internal value and
+    #   returning the value returned from the proc
+    #
     # @see http://wiki.commonjs.org/wiki/Promises/A
     # @see http://promises-aplus.github.io/promises-spec/
-    def initialize(options = {}, &block)
-      options.delete_if {|k, v| v.nil?}
+    def initialize(opts = {}, &block)
+      opts.delete_if {|k, v| v.nil?}
 
-      @parent = options.fetch(:parent) { nil }
-      @on_fulfill = options.fetch(:on_fulfill) { Proc.new{ |result| result } }
-      @on_reject = options.fetch(:on_reject) { Proc.new{ |reason| raise reason } }
+      @executor = get_executor_from(opts)
+      @parent = opts.fetch(:parent) { nil }
+      @on_fulfill = opts.fetch(:on_fulfill) { Proc.new{ |result| result } }
+      @on_reject = opts.fetch(:on_reject) { Proc.new{ |reason| raise reason } }
 
       @promise_body = block || Proc.new{|result| result }
       @state = :unscheduled
@@ -26,14 +47,14 @@ module Concurrent
     end
 
     # @return [Promise]
-    def self.fulfill(value)
-      Promise.new.tap { |p| p.send(:synchronized_set_state!, true, value, nil) }
+    def self.fulfill(value, opts = {})
+      Promise.new(opts).tap{ |p| p.send(:synchronized_set_state!, true, value, nil) }
     end
 
 
     # @return [Promise]
-    def self.reject(reason)
-      Promise.new.tap { |p| p.send(:synchronized_set_state!, false, nil, reason) }
+    def self.reject(reason, opts = {})
+      Promise.new(opts).tap{ |p| p.send(:synchronized_set_state!, false, nil, reason) }
     end
 
     # @return [Promise]
@@ -51,16 +72,20 @@ module Concurrent
     end
 
     # @since 0.5.0
-    def self.execute(&block)
-      new(&block).execute
+    def self.execute(opts = {}, &block)
+      new(opts, &block).execute
     end
 
-
     # @return [Promise] the new promise
     def then(rescuer = nil, &block)
       raise ArgumentError.new('rescuers and block are both missing') if rescuer.nil? && !block_given?
       block = Proc.new{ |result| result } if block.nil?
-      child = Promise.new(parent: self, on_fulfill: block, on_reject: rescuer)
+      child = Promise.new(
+        parent: self,
+        executor: @executor,
+        on_fulfill: block,
+        on_reject: rescuer
+      )
 
       mutex.synchronize do
         child.state = :pending if @state == :pending
@@ -118,7 +143,7 @@ module Concurrent
 
     # @!visibility private
     def realize(task)
-      Promise.thread_pool.post do
+      @executor.post do
         success, value, reason = SafeTaskExecutor.new( task ).execute
 
         children_to_notify = mutex.synchronize do
@@ -140,6 +165,5 @@ module Concurrent
         set_state!(success, value, reason)
       end
     end
-
   end
 end

data/lib/concurrent/ruby_cached_thread_pool.rb

@@ -0,0 +1,37 @@
+require 'concurrent/ruby_thread_pool_executor'
+
+module Concurrent
+
+  # @!macro cached_thread_pool
+  class RubyCachedThreadPool < RubyThreadPoolExecutor
+
+    # Create a new thread pool.
+    #
+    # @param [Hash] opts the options defining pool behavior.
+    # @option opts [Integer] :max_threads (+DEFAULT_MAX_POOL_SIZE+) maximum number
+    #   of threads which may be created in the pool
+    # @option opts [Integer] :idletime (+DEFAULT_THREAD_IDLETIMEOUT+) maximum
+    #   number of seconds a thread may be idle before it is reclaimed
+    #
+    # @raise [ArgumentError] if +max_threads+ is less than or equal to zero
+    # @raise [ArgumentError] if +idletime+ is less than or equal to zero
+    # @raise [ArgumentError] if +overflow_policy+ is not a known policy
+    def initialize(opts = {})
+      max_length = opts.fetch(:max_threads, DEFAULT_MAX_POOL_SIZE).to_i
+      idletime = opts.fetch(:idletime, DEFAULT_THREAD_IDLETIMEOUT).to_i
+      overflow_policy = opts.fetch(:overflow_policy, :abort)
+
+      raise ArgumentError.new('idletime must be greater than zero') if idletime <= 0
+      raise ArgumentError.new('max_threads must be greater than zero') if max_length <= 0
+      raise ArgumentError.new("#{overflow_policy} is not a valid overflow policy") unless OVERFLOW_POLICIES.include?(overflow_policy)
+
+      opts = opts.merge(
+        min_threads: 0,
+        max_threads: max_length,
+        num_threads: overflow_policy,
+        idletime: idletime
+      )
+      super(opts)
+    end
+  end
+end

data/lib/concurrent/ruby_fixed_thread_pool.rb

@@ -0,0 +1,31 @@
+require 'concurrent/ruby_thread_pool_executor'
+
+module Concurrent
+
+  # @!macro fixed_thread_pool
+  class RubyFixedThreadPool < RubyThreadPoolExecutor
+
+    # Create a new thread pool.
+    #
+    # @param [Integer] num_threads the number of threads to allocate
+    # @param [Hash] opts the options defining pool behavior.
+    # @option opts [Symbol] :overflow_policy (+:abort+) the overflow policy
+    #
+    # @raise [ArgumentError] if +num_threads+ is less than or equal to zero
+    # @raise [ArgumentError] if +overflow_policy+ is not a known policy
+    def initialize(num_threads, opts = {})
+      overflow_policy = opts.fetch(:overflow_policy, :abort)
+
+      raise ArgumentError.new('number of threads must be greater than zero') if num_threads < 1
+      raise ArgumentError.new("#{overflow_policy} is not a valid overflow policy") unless OVERFLOW_POLICIES.include?(overflow_policy)
+
+      opts = opts.merge(
+        min_threads: num_threads,
+        max_threads: num_threads,
+        num_threads: overflow_policy,
+        idletime: 0
+      )
+      super(opts)
+    end
+  end
+end

data/lib/concurrent/ruby_thread_pool_executor.rb

@@ -0,0 +1,268 @@
+require 'thread'
+
+require 'concurrent/event'
+require 'concurrent/ruby_thread_pool_worker'
+
+module Concurrent
+
+  RejectedExecutionError = Class.new(StandardError) unless defined? RejectedExecutionError
+
+  # @!macro thread_pool_executor
+  class RubyThreadPoolExecutor
+
+    # The maximum number of threads that will be created in the pool
+    # (unless overridden during construction).
+    DEFAULT_MAX_POOL_SIZE = 2**15 # 32768
+
+    # The minimum number of threads that will be created in the pool
+    # (unless overridden during construction).
+    DEFAULT_MIN_POOL_SIZE = 0
+
+    DEFAULT_MAX_QUEUE_SIZE = 0
+
+    # The maximum number of seconds a thread in the pool may remain idle before
+    # being reclaimed (unless overridden during construction).
+    DEFAULT_THREAD_IDLETIMEOUT = 60
+
+    OVERFLOW_POLICIES = [:abort, :discard, :caller_runs]
+
+    # The maximum number of threads that may be created in the pool.
+    attr_reader :max_length
+    attr_reader :min_length
+
+    attr_reader :largest_length
+
+    attr_reader :scheduled_task_count
+    attr_reader :completed_task_count
+
+    attr_reader :idletime
+
+    attr_reader :max_queue
+
+    attr_reader :overflow_policy
+
+    # Create a new thread pool.
+    #
+    # @see http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ThreadPoolExecutor.html
+    def initialize(opts = {})
+      @min_length = opts.fetch(:min_threads, DEFAULT_MIN_POOL_SIZE).to_i
+      @max_length = opts.fetch(:max_threads, DEFAULT_MAX_POOL_SIZE).to_i
+      @idletime = opts.fetch(:idletime, DEFAULT_THREAD_IDLETIMEOUT).to_i
+      @max_queue = opts.fetch(:max_queue, DEFAULT_MAX_QUEUE_SIZE).to_i
+      @overflow_policy = opts.fetch(:overflow_policy, :abort)
+
+      raise ArgumentError.new('max_threads must be greater than zero') if @max_length <= 0
+      raise ArgumentError.new('min_threads cannot be less than zero') if @min_length < 0
+      raise ArgumentError.new("#{overflow_policy} is not a valid overflow policy") unless OVERFLOW_POLICIES.include?(@overflow_policy)
+
+      @state = :running
+      @pool = []
+      @terminator = Event.new
+      @queue = Queue.new
+      @mutex = Mutex.new
+      @scheduled_task_count = 0
+      @completed_task_count = 0
+      @largest_length = 0
+
+      @gc_interval = opts.fetch(:gc_interval, 1).to_i # undocumented
+      @last_gc_time = Time.now.to_f - [1.0, (@gc_interval * 2.0)].max
+    end
+
+    def length
+      @mutex.synchronize do
+        @state != :shutdown ? @pool.length : 0
+      end
+    end
+    alias_method :current_length, :length
+
+    def queue_length
+      @queue.length
+    end
+
+    def remaining_capacity
+      @mutex.synchronize { @max_queue == 0 ? -1 : @max_queue - @queue.length }
+    end
+
+    # Is the thread pool running?
+    #
+    # @return [Boolean] +true+ when running, +false+ when shutting down or shutdown
+    def running?
+      @mutex.synchronize { @state == :running }
+    end
+
+    # Returns an array with the status of each thread in the pool
+    #
+    # This method is deprecated and will be removed soon.
+    def status
+      warn '[DEPRECATED] `status` is deprecated and will be removed soon.'
+      @mutex.synchronize { @pool.collect { |worker| worker.status } }
+    end
+    
+    # Is the thread pool shutdown?
+    #
+    # @return [Boolean] +true+ when shutdown, +false+ when shutting down or running
+    def shutdown?
+      @mutex.synchronize { @state != :running }
+    end
+
+    # Block until thread pool shutdown is complete or until +timeout+ seconds have
+    # passed.
+    #
+    # @note Does not initiate shutdown or termination. Either +shutdown+ or +kill+
+    #   must be called before this method (or on another thread).
+    #
+    # @param [Integer] timeout the maximum number of seconds to wait for shutdown to complete
+    #
+    # @return [Boolean] +true+ if shutdown complete or false on +timeout+
+    def wait_for_termination(timeout)
+      return @terminator.wait(timeout.to_i)
+    end
+
+    # Submit a task to the thread pool for asynchronous processing.
+    #
+    # @param [Array] args zero or more arguments to be passed to the task
+    #
+    # @yield the asynchronous task to perform
+    #
+    # @return [Boolean] +true+ if the task is queued, +false+ if the thread pool
+    #   is not running
+    #
+    # @raise [ArgumentError] if no task is given
+    def post(*args, &task)
+      raise ArgumentError.new('no block given') unless block_given?
+      @mutex.synchronize do
+        break false unless @state == :running
+        return handle_overflow(*args, &task) if @max_queue != 0 && @queue.length >= @max_queue
+        @scheduled_task_count += 1
+        @queue << [args, task]
+        if Time.now.to_f - @gc_interval >= @last_gc_time
+          prune_pool
+          @last_gc_time = Time.now.to_f
+        end
+        grow_pool
+        true
+      end
+    end
+
+    # Submit a task to the thread pool for asynchronous processing.
+    #
+    # @param [Proc] task the asynchronous task to perform
+    #
+    # @return [self] returns itself
+    def <<(task)
+      self.post(&task)
+      return self
+    end
+
+    # Begin an orderly shutdown. Tasks already in the queue will be executed,
+    # but no new tasks will be accepted. Has no additional effect if the
+    # thread pool is not running.
+    def shutdown
+      @mutex.synchronize do
+        break unless @state == :running
+        @queue.clear
+        if @pool.empty?
+          @state = :shutdown
+          @terminator.set
+        else
+          @state = :shuttingdown
+          @pool.length.times{ @queue << :stop }
+        end
+      end
+    end
+
+    # Begin an immediate shutdown. In-progress tasks will be allowed to
+    # complete but enqueued tasks will be dismissed and no new tasks
+    # will be accepted. Has no additional effect if the thread pool is
+    # not running.
+    def kill
+      @mutex.synchronize do
+        break if @state == :shutdown
+        @queue.clear
+        @state = :shutdown
+        drain_pool
+        @terminator.set
+      end
+    end
+
+    # @!visibility private
+    def on_end_task # :nodoc:
+      @mutex.synchronize do
+        @completed_task_count += 1 #if success
+        break unless @state == :running
+      end
+    end
+
+    # @!visibility private
+    def on_worker_exit(worker) # :nodoc:
+      @mutex.synchronize do
+        @pool.delete(worker)
+        if @pool.empty? && @state != :running
+          @state = :shutdown
+          @terminator.set
+        end
+      end
+    end
+
+    protected
+
+    # @!visibility private
+    def handle_overflow(*args) # :nodoc:
+      case @overflow_policy
+      when :abort
+        raise RejectedExecutionError
+      when :discard
+        false
+      when :caller_runs
+        begin
+          yield(*args)
+        rescue
+          # let it fail
+        end
+        true
+      end
+    end
+
+    # @!visibility private
+    def prune_pool # :nodoc:
+      @pool.delete_if do |worker|
+        worker.dead? ||
+          (@idletime == 0 ? false : Time.now.to_f - @idletime > worker.last_activity)
+      end
+    end
+
+    # @!visibility private
+    def grow_pool # :nodoc:
+      if @min_length > @pool.length
+        additional = @min_length - @pool.length
+      elsif @pool.length < @max_length && ! @queue.empty?
+        # NOTE: does not take into account idle threads
+        additional = 1
+      else
+        additional = 0
+      end
+      additional.times do
+        break if @pool.length >= @max_length
+        @pool << create_worker_thread
+      end
+      @largest_length = [@largest_length, @pool.length].max
+    end
+
+    # @!visibility private
+    def drain_pool # :nodoc:
+      @pool.each {|worker| worker.kill }
+      @pool.clear
+    end
+
+    # @!visibility private
+    def create_worker_thread # :nodoc:
+      wrkr = RubyThreadPoolWorker.new(@queue, self)
+      Thread.new(wrkr, self) do |worker, parent|
+        Thread.current.abort_on_exception = false
+        worker.run
+        parent.on_worker_exit(worker)
+      end
+      return wrkr
+    end
+  end
+end