concurrent-ruby 0.7.0 → 0.7.1

data/lib/concurrent/executor/indirect_immediate_executor.rb

@@ -0,0 +1,46 @@
+require 'concurrent/executor/executor'
+
+module Concurrent
+  # An executor service which runs all operations on a new thread, blocking
+  # until it completes. Operations are performed in the order they are received
+  # and no two operations can be performed simultaneously.
+  #
+  # This executor service exists mainly for testing an debugging. When used it
+  # immediately runs every `#post` operation on a new thread, blocking the
+  # current thread until the operation is complete. This is similar to how the
+  # ImmediateExecutor works, but the operation has the full stack of the new
+  # thread at its disposal. This can be helpful when the operations will spawn
+  # more operations on the same executor and so on - such a situation might
+  # overflow the single stack in case of an ImmediateExecutor, which is
+  # inconsistent with how it would behave for a threaded executor.
+  #
+  # @note Intended for use primarily in testing and debugging.
+  class IndirectImmediateExecutor < ImmediateExecutor
+    # Creates a new executor
+    def initialize
+      super
+      @internal_executor = PerThreadExecutor.new
+    end
+
+    # @!macro executor_method_post
+    def post(*args, &task)
+      raise ArgumentError.new("no block given") unless block_given?
+      return false unless running?
+
+      event = Concurrent::Event.new
+      internal_executor.post do
+        begin
+          task.call(*args)
+        ensure
+          event.set
+        end
+      end
+      event.wait
+
+      true
+    end
+
+    private
+    attr_reader :internal_executor
+  end
+end

data/lib/concurrent/executor/java_thread_pool_executor.rb

@@ -74,9 +74,7 @@ if RUBY_PLATFORM == 'java'
         raise ArgumentError.new('min_threads cannot be more than max_threads') if min_length > max_length
         raise ArgumentError.new("#{@overflow_policy} is not a valid overflow policy") unless OVERFLOW_POLICIES.keys.include?(@overflow_policy)
 
-        if min_length == 0 && @max_queue == 0
-          queue = java.util.concurrent.SynchronousQueue.new
-        elsif @max_queue == 0
+        if @max_queue == 0
           queue = java.util.concurrent.LinkedBlockingQueue.new
         else
           queue = java.util.concurrent.LinkedBlockingQueue.new(@max_queue)
@@ -90,7 +88,7 @@ if RUBY_PLATFORM == 'java'
         set_shutdown_hook
       end
 
-    # @!macro executor_module_method_can_overflow_question
+      # @!macro executor_module_method_can_overflow_question
       def can_overflow?
         @max_queue != 0
       end

data/lib/concurrent/executor/ruby_thread_pool_executor.rb

@@ -11,20 +11,20 @@ module Concurrent
     include RubyExecutor
 
     # Default maximum number of threads that will be created in the pool.
-    DEFAULT_MAX_POOL_SIZE = 2**15 # 32768
+    DEFAULT_MAX_POOL_SIZE      = 2**15 # 32768
 
     # Default minimum number of threads that will be retained in the pool.
-    DEFAULT_MIN_POOL_SIZE = 0
+    DEFAULT_MIN_POOL_SIZE      = 0
 
     # Default maximum number of tasks that may be added to the task queue.
-    DEFAULT_MAX_QUEUE_SIZE = 0
+    DEFAULT_MAX_QUEUE_SIZE     = 0
 
     # Default maximum number of seconds a thread in the pool may remain idle
     # before being reclaimed.
     DEFAULT_THREAD_IDLETIMEOUT = 60
 
     # The set of possible overflow policies that may be set at thread pool creation.
-    OVERFLOW_POLICIES = [:abort, :discard, :caller_runs]
+    OVERFLOW_POLICIES          = [:abort, :discard, :caller_runs]
 
     # The maximum number of threads that may be created in the pool.
     attr_reader :max_length
@@ -77,10 +77,10 @@ module Concurrent
     #
     # @see http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ThreadPoolExecutor.html
     def initialize(opts = {})
-      @min_length = opts.fetch(:min_threads, DEFAULT_MIN_POOL_SIZE).to_i
-      @max_length = opts.fetch(:max_threads, DEFAULT_MAX_POOL_SIZE).to_i
-      @idletime = opts.fetch(:idletime, DEFAULT_THREAD_IDLETIMEOUT).to_i
-      @max_queue = opts.fetch(:max_queue, DEFAULT_MAX_QUEUE_SIZE).to_i
+      @min_length      = opts.fetch(:min_threads, DEFAULT_MIN_POOL_SIZE).to_i
+      @max_length      = opts.fetch(:max_threads, DEFAULT_MAX_POOL_SIZE).to_i
+      @idletime        = opts.fetch(:idletime, DEFAULT_THREAD_IDLETIMEOUT).to_i
+      @max_queue       = opts.fetch(:max_queue, DEFAULT_MAX_QUEUE_SIZE).to_i
       @overflow_policy = opts.fetch(:overflow_policy, :abort)
 
       raise ArgumentError.new('max_threads must be greater than zero') if @max_length <= 0
@@ -90,13 +90,13 @@ module Concurrent
 
       init_executor
 
-      @pool = []
-      @queue = Queue.new
+      @pool                 = []
+      @queue                = Queue.new
       @scheduled_task_count = 0
       @completed_task_count = 0
-      @largest_length = 0
+      @largest_length       = 0
 
-      @gc_interval = opts.fetch(:gc_interval, 1).to_i # undocumented
+      @gc_interval  = opts.fetch(:gc_interval, 1).to_i # undocumented
       @last_gc_time = Time.now.to_f - [1.0, (@gc_interval * 2.0)].max
     end
 
@@ -109,15 +109,16 @@ module Concurrent
     #
     # @return [Integer] the length
     def length
-      mutex.synchronize{ running? ? @pool.length : 0 }
+      mutex.synchronize { running? ? @pool.length : 0 }
     end
+
     alias_method :current_length, :length
 
     # The number of tasks in the queue awaiting execution.
     #
     # @return [Integer] the queue_length
     def queue_length
-      mutex.synchronize{ running? ? @queue.length : 0 }
+      mutex.synchronize { running? ? @queue.length : 0 }
     end
 
     # Number of tasks that may be enqueued before reaching `max_queue` and rejecting
@@ -152,7 +153,7 @@ module Concurrent
     def on_worker_exit(worker)
       mutex.synchronize do
         @pool.delete(worker)
-        if @pool.empty? && ! running?
+        if @pool.empty? && !running?
           stop_event.set
           stopped_event.set
         end
@@ -177,7 +178,7 @@ module Concurrent
       if @pool.empty?
         stopped_event.set
       else
-        @pool.length.times{ @queue << :stop }
+        @pool.length.times { @queue << :stop }
       end
     end
 
@@ -196,7 +197,7 @@ module Concurrent
     # @!visibility private
     def ensure_capacity?
       additional = 0
-      capacity = true
+      capacity   = true
 
       if @pool.size < @min_length
         additional = @min_length - @pool.size
@@ -254,10 +255,11 @@ module Concurrent
     # @!visibility private
     def prune_pool
       if Time.now.to_f - @gc_interval >= @last_gc_time
-        @pool.delete_if do |worker|
-          worker.dead? ||
-            (@idletime == 0 ? false : Time.now.to_f - @idletime > worker.last_activity)
-        end
+        @pool.delete_if { |worker| worker.dead? }
+        # send :stop for each thread over idletime
+        @pool.
+            select { |worker| @idletime != 0 && Time.now.to_f - @idletime > worker.last_activity }.
+            each { @queue << :stop }
         @last_gc_time = Time.now.to_f
       end
     end
@@ -266,7 +268,7 @@ module Concurrent
     #
     # @!visibility private
     def drain_pool
-      @pool.each {|worker| worker.kill }
+      @pool.each { |worker| worker.kill }
       @pool.clear
     end
 

data/lib/concurrent/executor/thread_pool_executor.rb

@@ -49,6 +49,8 @@ module Concurrent
     #   * `:discard`: Silently discard the task and return `nil` as the task result.
     #   * `:caller_runs`: Execute the task on the calling thread.
     #
+    #   {include:file:doc/thread_pools.md}
+    #
     #   @note When running on the JVM (JRuby) this class will inherit from `JavaThreadPoolExecutor`.
     #     On all other platforms it will inherit from `RubyThreadPoolExecutor`.
     #

data/lib/concurrent/executor/timer_set.rb

@@ -22,10 +22,10 @@ module Concurrent
     # @option opts [object] :executor when provided will run all operations on
     #   this executor rather than the global thread pool (overrides :operation)
     def initialize(opts = {})
-      @queue = PriorityQueue.new(order: :min)
-      @task_executor = OptionsParser::get_executor_from(opts) || Concurrent.configuration.global_task_pool
+      @queue          = PriorityQueue.new(order: :min)
+      @task_executor  = OptionsParser::get_executor_from(opts) || Concurrent.configuration.global_task_pool
       @timer_executor = SingleThreadExecutor.new
-      @condition = Condition.new
+      @condition      = Condition.new
       init_executor
     end
 
@@ -64,7 +64,7 @@ module Concurrent
     # For a timer, #kill is like an orderly shutdown, except we need to manually
     # (and destructively) clear the queue first
     def kill
-      @queue.clear
+      mutex.synchronize { @queue.clear }
       shutdown
     end
 
@@ -124,14 +124,13 @@ module Concurrent
     # @!visibility private
     def process_tasks
       loop do
-        break if @queue.empty?
-
-        task = @queue.peek
+        task = mutex.synchronize { @queue.peek }
+        break unless task
         interval = task.time - Time.now.to_f
 
         if interval <= 0
           @task_executor.post(*task.args, &task.op)
-          @queue.pop
+          mutex.synchronize { @queue.pop }
         else
           mutex.synchronize do
             @condition.wait(mutex, [interval, 60].min)

data/lib/concurrent/executors.rb

@@ -1,6 +1,7 @@
 require 'concurrent/executor/cached_thread_pool'
 require 'concurrent/executor/fixed_thread_pool'
 require 'concurrent/executor/immediate_executor'
+require 'concurrent/executor/indirect_immediate_executor'
 require 'concurrent/executor/per_thread_executor'
 require 'concurrent/executor/safe_task_executor'
 require 'concurrent/executor/single_thread_executor'

data/lib/concurrent/future.rb

@@ -6,35 +6,7 @@ require 'concurrent/executor/safe_task_executor'
 
 module Concurrent
 
-  # A `Future` represents a promise to complete an action at some time in the future.
-  # The action is atomic and permanent. The idea behind a future is to send an operation
-  # for asynchronous completion, do other stuff, then return and retrieve the result
-  # of the async operation at a later time.
-  #
-  # A `Future` has four possible states: *:unscheduled*, *:pending*, *:rejected*, or *:fulfilled*.
-  # When a `Future` is created its state is set to *:unscheduled*. Once the `#execute` method is
-  # called the state becomes *:pending* and will remain in that state until processing is
-  # complete. A completed `Future` is either *:rejected*, indicating that an exception was
-  # thrown during processing, or *:fulfilled*, indicating success. If a `Future` is *:fulfilled*
-  # its `value` will be updated to reflect the result of the operation. If *:rejected* the
-  # `reason` will be updated with a reference to the thrown exception. The predicate methods
-  # `#unscheduled?`, `#pending?`, `#rejected?`, and `fulfilled?` can be called at any time to
-  # obtain the state of the `Future`, as can the `#state` method, which returns a symbol. 
-  #
-  # Retrieving the value of a `Future` is done through the `#value` (alias: `#deref`) method.
-  # Obtaining the value of a `Future` is a potentially blocking operation. When a `Future` is
-  # *:rejected* a call to `#value` will return `nil` immediately. When a `Future` is
-  # *:fulfilled* a call to `#value` will immediately return the current value. When a
-  # `Future` is *:pending* a call to `#value` will block until the `Future` is either
-  # *:rejected* or *:fulfilled*. A *timeout* value can be passed to `#value` to limit how
-  # long the call will block. If `nil` the call will block indefinitely. If `0` the call will
-  # not block. Any other integer or float value will indicate the maximum number of seconds to block.
-  #
-  # The `Future` class also includes the behavior of the Ruby standard library `Observable` module,
-  # but does so in a thread-safe way. On fulfillment or rejection all observers will be notified
-  # according to the normal `Observable` behavior. The observer callback function will be called
-  # with three parameters: the `Time` of fulfillment/rejection, the final `value`, and the final
-  # `reason`. Observers added after fulfillment/rejection will still be notified as normal.
+  # {include:file:doc/future.md}
   #
   # @see http://ruby-doc.org/stdlib-2.1.1/libdoc/observer/rdoc/Observable.html Ruby Observable module
   # @see http://clojuredocs.org/clojure_core/clojure.core/future Clojure's future function
@@ -94,6 +66,12 @@ module Concurrent
     #
     # @yield the asynchronous operation to perform
     #
+    # @param [Hash] opts the options controlling how the future will be processed
+    # @option opts [Boolean] :operation (false) when `true` will execute the future on the global
+    #   operation pool (for long-running operations), when `false` will execute the future on the
+    #   global task pool (for short-running tasks)
+    # @option opts [object] :executor when provided will run all operations on
+    #   this executor rather than the global thread pool (overrides :operation)
     # @option opts [String] :dup_on_deref (false) call `#dup` before returning the data
     # @option opts [String] :freeze_on_deref (false) call `#freeze` before returning the data
     # @option opts [String] :copy_on_deref (nil) call the given `Proc` passing the internal value and

data/lib/concurrent/ivar.rb

@@ -6,6 +6,10 @@ require 'concurrent/observable'
 
 module Concurrent
 
+  # An `IVar` is like a future that you can assign. As a future is a value that is being computed that you can wait on, an `IVar` is a value that is waiting to be assigned, that you can wait on. `IVars` are single assignment and deterministic.
+  # 
+  # Then, express futures as an asynchronous computation that assigns an `IVar`. The `IVar` becomes the primitive on which [futures](Future) and [dataflow](Dataflow) are built.
+  #
   # An `IVar` is a single-element container that is normally created empty, and
   # can only be set once. The I in `IVar` stands for immutable. Reading an `IVar`
   # normally blocks until it is set. It is safe to set and read an `IVar` from
@@ -15,6 +19,11 @@ module Concurrent
   # a `Future`. If you want to create a graph of parallel tasks all executed when
   # the values they depend on are ready you want `dataflow`. `IVar` is generally
   # a low-level primitive.
+  # 
+  # **See Also:**
+  #
+  # * For the theory: Arvind, R. Nikhil, and K. Pingali. [I-Structures: Data structures for parallel computing](http://dl.acm.org/citation.cfm?id=69562). In Proceedings of Workshop on Graph Reduction, 1986.
+  # * For recent application: [DataDrivenFuture in Habanero Java from Rice](http://www.cs.rice.edu/~vs3/hjlib/doc/edu/rice/hj/api/HjDataDrivenFuture.html).
   #
   # @example Create, set and get an `IVar`
   #   ivar = Concurrent::IVar.new

data/lib/concurrent/logging.rb

@@ -12,6 +12,9 @@ module Concurrent
     # @yieldreturn [String] a message
     def log(level, progname, message = nil, &block)
       (@logger || Concurrent.configuration.logger).call level, progname, message, &block
+    rescue => error
+      $stderr.puts "`Concurrent.configuration.logger` failed to log #{[level, progname, message, block]}\n" +
+          "#{error.message} (#{error.class})\n#{error.backtrace.join "\n"}"
     end
   end
 end

data/lib/concurrent/mvar.rb

@@ -4,16 +4,33 @@ require 'concurrent/atomic/event'
 
 module Concurrent
 
-  # An `MVar` is a single-element container that blocks on `get` if it is empty,
-  # and blocks on `put` if it is full. It is safe to use an `MVar` from
-  # multiple threads. `MVar` can be seen as a single-element blocking queue, or
-  # a rendezvous variable.
+  # An `MVar` is a synchronized single element container. They are empty or contain one item.
+  # Taking a value from an empty `MVar` blocks, as does putting a value into a full one.
+  # You can either think of them as blocking queue of length one, or a special kind of
+  # mutable variable. 
+  # 
+  # On top of the fundamental `#put` and `#take` operations, we also provide a `#mutate`
+  # that is atomic with respect to operations on the same instance. These operations all
+  # support timeouts. 
+  # 
+  # We also support non-blocking operations `#try_put!` and `#try_take!`, a `#set!` that
+  # ignores existing values, a `#value` that returns the value without removing it or
+  # returns `MVar::EMPTY`, and a `#modify!` that yields `MVar::EMPTY` if the `MVar` is
+  # empty and can be used to set `MVar::EMPTY`. You shouldn't use these operations in the
+  # first instance. 
+  # 
+  # `MVar` is a [Dereferenceable](Dereferenceable).
+  # 
+  # `MVar` is related to M-structures in Id, `MVar` in Haskell and `SyncVar` in Scala.
   #
-  # An `MVar` is typically used to transfer objects between threads, where the
-  # sending thread will block if the previous message hasn't been taken yet by the
-  # receiving thread. It can also be used to control access to some global shared
-  # state, where threads `take` the value, perform some operation, and then
-  # `put` it back.
+  # Note that unlike the original Haskell paper, our `#take` is blocking. This is how
+  # Haskell and Scala do it today.
+  # 
+  # **See Also:**
+  # 
+  # 1. P. Barth, R. Nikhil, and Arvind. [M-Structures: Extending a parallel, non-
+  # strict, functional language with state](http://dl.acm.org/citation.cfm?id=652538). In Proceedings of the 5th ACM Conference on Functional Programming Languages and Computer Architecture (FPCA), 1991.
+  # 2. S. Peyton Jones, A. Gordon, and S. Finne. [Concurrent Haskell](http://dl.acm.org/citation.cfm?id=237794). In Proceedings of the 23rd Symposium on Principles of Programming Languages (PoPL), 1996.
   class MVar
 
     include Dereferenceable

data/lib/concurrent/observable.rb

@@ -3,6 +3,39 @@ require 'concurrent/atomic/copy_on_write_observer_set'
 
 module Concurrent
 
+  # The [observer pattern](http://en.wikipedia.org/wiki/Observer_pattern) is one of the most useful design pattern.
+  # 
+  # The workflow is very simple:
+  # - an `observer` can register itself to a `subject` via a callback
+  # - many `observers` can be registered to the same `subject`
+  # - the `subject` notifies all registered observers when its status changes
+  # - an `observer` can deregister itself when is no more interested to receive event notifications
+  # 
+  # In a single threaded environment the whole pattern is very easy: the `subject` can use a simple data structure to manage all its subscribed `observer`s and every `observer` can react directly to every event without caring about synchronization.
+  # 
+  # In a multi threaded environment things are more complex.
+  # The `subject` must synchronize the access to its data structure and to do so currently we're using two specialized ObserverSet: CopyOnWriteObserverSet and CopyOnNotifyObserverSet.
+  # 
+  # When implementing and `observer` there's a very important rule to remember: **there are no guarantees about the thread that will execute the callback**
+  # 
+  # Let's take this example
+  # ```
+  # class Observer
+  #   def initialize
+  #     @count = 0
+  #   end
+  # 
+  #   def update
+  #     @count += 1
+  #   end
+  # end
+  # 
+  # obs = Observer.new
+  # [obj1, obj2, obj3, obj4].each { |o| o.add_observer(obs) }
+  # # execute [obj1, obj2, obj3, obj4]
+  # ```
+  # 
+  # `obs` is wrong because the variable `@count` can be accessed by different threads at the same time, so it should be synchronized (using either a Mutex or an AtomicFixum)
   module Observable
 
     # @return [Object] the added observer

data/lib/concurrent/promise.rb

@@ -5,8 +5,9 @@ require 'concurrent/options_parser'
 
 module Concurrent
 
-  # TODO unify promise and future to single class, with dataflow
+  # {include:file:doc/promise.md}
   class Promise
+    # TODO unify promise and future to single class, with dataflow
     include Obligation
 
     # Initialize a new Promise with the provided options.
@@ -110,6 +111,63 @@ module Concurrent
     alias_method :catch, :rescue
     alias_method :on_error, :rescue
 
+    # Yield the successful result to the block that returns a promise. If that
+    # promise is also successful the result is the result of the yielded promise.
+    # If either part fails the whole also fails.
+    #
+    # @example
+    #   Promise.execute { 1 }.flat_map { |v| Promise.execute { v + 2 } }.value! #=> 3
+    #
+    # @return [Promise]
+    def flat_map(&block)
+      child = Promise.new(
+        parent: self,
+        executor: ImmediateExecutor.new,
+      )
+
+      on_error { |e| child.on_reject(e) }
+      on_success do |result1|
+        begin
+          inner = block.call(result1)
+          inner.execute
+          inner.on_success { |result2| child.on_fulfill(result2) }
+          inner.on_error { |e| child.on_reject(e) }
+        rescue => e
+          child.on_reject(e)
+        end
+      end
+
+      child
+    end
+
+    # Builds a promise that produces the result of promises in an Array
+    # and fails if any of them fails.
+    #
+    # @param [Array<Promise>] promises
+    #
+    # @return [Promise<Array>]
+    def self.zip(*promises)
+      zero = fulfill([], executor: ImmediateExecutor.new)
+
+      promises.reduce(zero) do |p1, p2|
+        p1.flat_map do |results|
+          p2.then do |next_result|
+            results << next_result
+          end
+        end
+      end
+    end
+
+    # Builds a promise that produces the result of self and others in an Array
+    # and fails if any of them fails.
+    #
+    # @param [Array<Promise>] others
+    #
+    # @return [Promise<Array>]
+    def zip(*others)
+      self.class.zip(self, *others)
+    end
+
     protected
 
     def set_pending