procrastinate 0.2.0 → 0.3.0

data/README

@@ -4,13 +4,13 @@ INTRO
 to concentrate on what to run when, not orchestration of low level details. 
 
 This library will be ideal for quickly scheduling of a lot of long running
-tasks. You can easily control how many processes are run at any time. To get
-at/cron like scheduling, combine this library with 'rufus-scheduler'. 
+tasks. You can easily control how many processes are run at any time. Your 
+main thread can continue to do useful work until it accesses the results of
+the computation, at which point it will wait for the processes to finish. 
 
 SYNOPSIS
 
-  require 'procrastinate'
-  include Procrastinate
+  require 'procrastinate/implicit'
 
   class Worker
     def do_work
@@ -20,43 +20,42 @@ SYNOPSIS
     end
   end
 
-  scheduler = Scheduler.start(SpawnStrategy::Throttled.new(5))
-  worker = scheduler.create_proxy(Worker.new)
+  worker = Procrastinate.proxy(Worker.new)
 
   10.times do 
     worker.do_work
   end
 
-  scheduler.shutdown
-  
+  Procrastinate.join
+
 The above example will output something like 
 
-  > Starting work in process 6651
-  > Starting work in process 6652
-  > Starting work in process 6653
-  > Starting work in process 6654
-  > Starting work in process 6655
-  < Work completed in process 6651
-  < Work completed in process 6652
-  < Work completed in process 6653
-  < Work completed in process 6654
-  < Work completed in process 6655
-  > Starting work in process 6659
-  > Starting work in process 6660
-  > Starting work in process 6656
-  > Starting work in process 6657
-  > Starting work in process 6658
-  < Work completed in process 6659
-  < Work completed in process 6660
-  < Work completed in process 6656
-  < Work completed in process 6657
-  < Work completed in process 6658
+  > Starting work in process 56144
+  > Starting work in process 56145
+  > Starting work in process 56146
+  > Starting work in process 56147
+  > Starting work in process 56148
+  > Starting work in process 56149
+  < Work completed in process 56144
+  < Work completed in process 56145
+  < Work completed in process 56146
+  < Work completed in process 56147
+  < Work completed in process 56148
+  < Work completed in process 56149
+  > Starting work in process 56150
+  > Starting work in process 56151
+  > Starting work in process 56152
+  > Starting work in process 56153
+  < Work completed in process 56150
+  < Work completed in process 56151
+  < Work completed in process 56152
+  < Work completed in process 56153
+  
+(The output depends on the number of cores your machine has)
 
 COMPATIBILITY
 
-This library runs with at least Ruby 1.8.7 and Ruby 1.9. Ruby 1.9 support
-might be spotty, because the threading primitives in Ruby 1.9 are still 
-somewhat buggy. 
+This library runs with at least Ruby 1.8.7 and Ruby 1.9. 
 
 KNOWN BUGS
 

data/Rakefile

@@ -16,7 +16,7 @@ spec = Gem::Specification.new do |s|
 
   # Change these as appropriate
   s.name              = "procrastinate"
-  s.version           = "0.2.0"
+  s.version           = "0.3.0"
   s.summary           = "Framework to run tasks in separate processes."
   s.authors           = ['Kaspar Schiess', 'Patrick Marchi']
   s.email             = ['kaspar.schiess@absurd.li', 'mail@patrickmarchi.ch']

data/lib/procrastinate/implicit.rb

@@ -0,0 +1,58 @@
+
+require 'procrastinate'
+
+module Procrastinate
+  # call-seq:
+  #   Procrastinate.scheduler => scheduler
+  #
+  # Returns the scheduler instance. When using procrastinate/implicit, there
+  # is one global scheduler to your ruby process, this one.
+  #
+  def scheduler
+    @scheduler ||= Scheduler.start
+  end
+  module_function :scheduler
+  
+  # call-seq: 
+  #   Procrastinate.proxy(obj) => proxy
+  #
+  # Creates a proxy that will execute methods called on obj in a child process. 
+  #
+  # Example: 
+  #
+  #   proxy = Procrastinate.proxy("foo")
+  #   r     = proxy += "bar"
+  #   r.value   # => 'foobar'
+  #
+  def proxy(obj)
+    scheduler.proxy(obj)
+  end
+  module_function :proxy
+  
+  # call-seq: 
+  #   Procrastinate.join
+  #
+  # Waits for all currently scheduled tasks to complete. This is like calling
+  # #value on all result objects, except that nothing is returned.
+  #
+  # Example: 
+  #
+  #   proxy = Procrastinate.proxy("foo")
+  #   r     = proxy += "bar"
+  #   Procrastinate.join
+  #   r.ready? # => true
+  #
+  def join
+    scheduler.join
+  end
+  module_function :join
+  
+  # Internal method: You should not have to shutdown the scheduler manually
+  # since it consumes almost no resources when not active. This is mainly
+  # useful in tests to achieve isolation. 
+  #
+  def shutdown
+    scheduler.shutdown
+  end
+  module_function :shutdown
+end

data/lib/procrastinate/process_manager.rb

@@ -8,6 +8,9 @@ require 'state_machine'
 class Procrastinate::ProcessManager
   include Procrastinate::IPC
   
+  autoload :ObjectEndpoint, 'procrastinate/process_manager/object_endpoint'
+  autoload :ChildProcess,   'procrastinate/process_manager/child_process'
+  
   # This pipe is used to wait for events in the master process. 
   attr_reader :control_pipe
   
@@ -15,45 +18,7 @@ class Procrastinate::ProcessManager
   # processes we spawn. Once the process is complete, the callback is called
   # in the procrastinate thread.
   attr_reader :children
-  
-  # A class that acts as a filter between ProcessManager and the endpoint it
-  # uses to communicate with its children. This converts Ruby objects into
-  # Strings and also sends process id. 
-  #
-  class ObjectEndpoint < Struct.new(:endpoint, :pid)
-    def send(obj)
-      msg = Marshal.dump([pid, obj])
-      endpoint.send(msg)
-    end
-  end
-  
-  # A <completion handler, result> tuple that stores the handler to call when
-  # a child exits and the object that will handle child-master communication
-  # if desired.
-  #
-  class Child < Struct.new(:handler, :result, :state)
-    state_machine :state, :initial => :new do
-      event(:start) { transition :new => :running }
-      event(:died)  { transition :running => :dead }
       
-      after_transition :on => :died, :do => :call_completion_handlers
-    end
-    
-    # Calls the completion handler for the child. This is triggered by the
-    # transition into the 'dead' state. 
-    #
-    def call_completion_handlers
-      result.process_died if result
-      handler.call if handler
-    end
-        
-    # Handles incoming messages from the tasks process.
-    #
-    def incoming_message(obj)
-      result.incoming_message(obj) if result
-    end
-  end
-  
   def initialize
     # This controls process manager wakeup
     @control_pipe = IO.pipe
@@ -101,6 +66,14 @@ class Procrastinate::ProcessManager
     # Ignore:
   end
   
+  # Returns the number of child processes that are alive at this point. Note
+  # that even if a child process is marked dead internally, it counts towards
+  # this number, since its results may not have been dispatched yet. 
+  # 
+  def process_count
+    children.count
+  end
+  
   # Internal methods below this point. ---------------------------------------
 
   # Register signals that aid in child care. NB: Because we do this globally, 
@@ -221,7 +194,7 @@ class Procrastinate::ProcessManager
     # The spawning is done in the same thread as the reaping is done. This is 
     # why no race condition to the following line exists. (or in other code, 
     # for that matter.)
-    children[pid] = Child.new(completion_handler, result).tap { |s| s.start }
+    children[pid] = ChildProcess.new(completion_handler, result).tap { |s| s.start }
   end
   
   # Gets executed in child process to clean up file handles and pipes that the

data/lib/procrastinate/process_manager/child_process.rb

@@ -0,0 +1,26 @@
+# A <completion handler, result> tuple that stores the handler to call when
+# a child exits and the object that will handle child-master communication
+# if desired.
+#
+class Procrastinate::ProcessManager::ChildProcess < Struct.new(:handler, :result, :state)
+  state_machine :state, :initial => :new do
+    event(:start) { transition :new => :running }
+    event(:died)  { transition :running => :dead }
+    
+    after_transition :on => :died, :do => :call_completion_handlers
+  end
+  
+  # Calls the completion handler for the child. This is triggered by the
+  # transition into the 'dead' state. 
+  #
+  def call_completion_handlers
+    result.process_died if result
+    handler.call if handler
+  end
+      
+  # Handles incoming messages from the tasks process.
+  #
+  def incoming_message(obj)
+    result.incoming_message(obj) if result
+  end
+end

data/lib/procrastinate/process_manager/object_endpoint.rb

@@ -0,0 +1,10 @@
+# A class that acts as a filter between ProcessManager and the endpoint it
+# uses to communicate with its children. This converts Ruby objects into
+# Strings and also sends process id. 
+#
+class Procrastinate::ProcessManager::ObjectEndpoint < Struct.new(:endpoint, :pid)
+  def send(obj)
+    msg = Marshal.dump([pid, obj])
+    endpoint.send(msg)
+  end
+end

data/lib/procrastinate/proxy.rb

@@ -4,7 +4,7 @@
 class Procrastinate::Proxy
   # Create a new proxy class. +worker+ is an instance of the class that we 
   # want to perform work in, +scheduler+ is where the work will be scheduled. 
-  # Don't call this on your own, instead use Scheduler#create_proxy. 
+  # Don't call this on your own, instead use Scheduler#proxy. 
   #
   def initialize(worker, scheduler) # :nodoc: 
     @worker = worker

data/lib/procrastinate/scheduler.rb

@@ -14,7 +14,7 @@ class Procrastinate::Scheduler
   attr_reader :task_queue
     
   def initialize(strategy)
-    @strategy   = strategy || Procrastinate::SpawnStrategy::Simple.new
+    @strategy   = strategy || Procrastinate::SpawnStrategy::Default.new
     @manager = Procrastinate::ProcessManager.new
 
     # State takes three values: :running, :soft_shutdown, :real_shutdown
@@ -39,10 +39,10 @@ class Procrastinate::Scheduler
   #
   # Example: 
   #
-  #   proxy = scheduler.create_proxy(worker)
+  #   proxy = scheduler.proxy(worker)
   #   status = proxy.do_some_work    # will execute later and in its own process
   #
-  def create_proxy(worker)
+  def proxy(worker)
     return Procrastinate::Proxy.new(worker, self)
   end
   
@@ -65,17 +65,31 @@ class Procrastinate::Scheduler
     manager.wakeup
   end
   
+  # Waits for the currently queued work to complete. This can be used at the
+  # end of short scripts to ensure that all work is done. 
+  #
+  def join
+    @state = :soft_shutdown
+    
+    # NOTE: Currently, this method busy-loops until all childs terminate. 
+    # This is not as elegant as I whish it to be, but its a start. 
+    
+    # Wait until all tasks are done.
+    loop do
+      manager.wakeup
+      break if task_queue.empty? && manager.process_count==0
+      sleep 0.01
+    end
+    
+  ensure
+    @state = :running
+  end
+  
   # Immediately shuts down the procrastinate thread and frees resources. 
   # If there are any tasks left in the queue, they will NOT be executed. 
   #
   def shutdown(hard=false)
-    unless hard
-      @state = :soft_shutdown
-      loop do
-        manager.wakeup
-        break if task_queue.empty?
-      end
-    end
+    join unless hard
     
     # Set the flag that will provoke shutdown
     @state = :real_shutdown
@@ -87,21 +101,22 @@ class Procrastinate::Scheduler
   end
   
 private
-  # Spawns new tasks (if needed). This is only ever called from the control
-  # thread (see below). 
-  # 
+  # Spawns new tasks (if needed). 
+  #   *control thread* 
+  #
   def spawn
     while strategy.should_spawn? && !task_queue.empty?
       task = task_queue.pop
+      strategy.notify_spawn
       manager.create_process(task) do
         strategy.notify_dead
       end
-      strategy.notify_spawn
     end
   end
   
   # This is the content of the control thread that is spawned with
   # #start_thread
+  #   *control thread* 
   #
   def run
     # Start managers work

data/lib/procrastinate/spawn_strategy.rb

@@ -2,4 +2,5 @@
 module Procrastinate::SpawnStrategy
   autoload :Simple, 'procrastinate/spawn_strategy/simple'
   autoload :Throttled, 'procrastinate/spawn_strategy/throttled'
+  autoload :Default, 'procrastinate/spawn_strategy/default'
 end

data/lib/procrastinate/spawn_strategy/default.rb

@@ -0,0 +1,31 @@
+
+# A dispatcher strategy that throttles tasks starting and ensures that no more
+# than limit processes run concurrently. Limit is initialized to the number of
+# cores this system has (+1) or a default value of 5 processes if the number
+# of cores cannot be autodetected.
+#
+class Procrastinate::SpawnStrategy::Default < Procrastinate::SpawnStrategy::Throttled
+  def initialize(workload_factor=3)
+    # In reality, this depends on what workload you have. You might want to 
+    # tune this number.
+    super(autodetect_cores*workload_factor)
+  end
+  
+  def autodetect_cores
+    # Linux / all OS with a /proc filesystem
+    if File.exist?('/proc/cpuinfo')
+      return Integer(`cat /proc/cpuinfo | grep "processor" | wc -l`.chomp)
+    end
+      
+    # Mac OS X 
+    if File.exist?('/usr/sbin/system_profiler')
+      output = `system_profiler SPHardwareDataType | grep "Total Number Of Cores"`
+      if md=output.match(%r(Total Number Of Cores: (\d+)))
+        return Integer(md[1])
+      end
+    end
+    
+    warn "Could not detect the number of CPU cores. Using a default of 2."
+    return 2
+  end
+end

data/lib/procrastinate/task/result.rb

@@ -12,6 +12,7 @@ class Procrastinate::Task::Result
   end
   
   # Gets passed all messages sent by the child process for this task.
+  #   *control thread* 
   #
   def incoming_message(obj)
     return if ready?
@@ -23,6 +24,7 @@ class Procrastinate::Task::Result
   # Notifies this result that the process has died. If this happens before
   # a process result is passed to #incoming_message, that message will never
   # arrive. 
+  #   *control thread* 
   #
   def process_died
     return if ready?

data/lib/procrastinate/utils/one_time_flag_ruby18_shim.rb

@@ -1,7 +1,5 @@
 class Procrastinate::Utils::OneTimeFlag
   def initialize
-    @waiting_m  = Mutex.new
-    @waiting_cv = ConditionVariable.new
     @set        = false
   end
   
@@ -10,18 +8,13 @@ class Procrastinate::Utils::OneTimeFlag
   def wait
     return if set?
     
-    @waiting_m.synchronize do
-      @waiting_cv.wait(@waiting_m)
-    end
+    sleep(0.01) until set?
   end
   
   # Sets the flag and releases all waiting threads.
   #
   def set
     @set = true
-    @waiting_m.synchronize do
-      @waiting_cv.broadcast
-    end
   end
   
   # Non blocking: Is the flag set?

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Kaspar Schiess
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-22 00:00:00 +01:00
+date: 2010-12-27 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -73,13 +73,17 @@ files:
 - LICENSE
 - Rakefile
 - README
+- lib/procrastinate/implicit.rb
 - lib/procrastinate/ipc/endpoint.rb
 - lib/procrastinate/ipc.rb
 - lib/procrastinate/lock.rb
+- lib/procrastinate/process_manager/child_process.rb
+- lib/procrastinate/process_manager/object_endpoint.rb
 - lib/procrastinate/process_manager.rb
 - lib/procrastinate/proxy.rb
 - lib/procrastinate/runtime.rb
 - lib/procrastinate/scheduler.rb
+- lib/procrastinate/spawn_strategy/default.rb
 - lib/procrastinate/spawn_strategy/simple.rb
 - lib/procrastinate/spawn_strategy/throttled.rb
 - lib/procrastinate/spawn_strategy.rb