procrastinate 0.3.1 → 0.4.0

data/HISTORY.txt

@@ -0,0 +1,37 @@
+= 0.4.0 / ???
+
+  + Schedules blocks as well (Scheduler#schedule).
+
+  ! Fixes a small timing bug that would provoke a ChildDeath where the child
+    exited correctly in reality.
+
+  ! Fixes a data race on the result variable. This would have your process
+    hang forever waiting for a result that was already available.
+
+= 0.3.1 / 13Sep2011 
+
+  ! fix processor detection for Mac OS X Lion
+
+= 0.3.0 / 27Dez2010
+  * create_proxy is now just 'proxy' to friends. This looks cleaner. 
+  
+  * require 'procrastinate/implicit' allows ignoring the scheduler in daily
+    usage. This is probably what will be more common.
+    
+  * Auto-detection of the number of cores procrastinate runs on.    
+    (SpawnStrategy::Default)
+    
+  * Ruby 1.8 now uses absolutely poor busy-loop synchs. This is still better
+    than inserting puts in the code to unblock some magical mystical internal
+    state. I'll try to find a better solution. 
+
+= 0.2.0 / 22Dez2010
+  * Big rewrite, trying to make things more clear. Also: There might be more
+    features lurking in there.
+    
+  * All proxy method calls now have a return value that works like a future: 
+    Access to the real return value through future.value, which might block.
+
+= 0.1.0 / 10Dez2010
+
+  * Initial version

data/README

@@ -60,6 +60,10 @@ This library runs with at least Ruby 1.8.7 and Ruby 1.9.
 Ruby 1.9-p136 users must use this patch: 
 https://gist.github.com/762807
 
+As a general remark: Interaction with Ruby versions is significant. Please 
+use the latest version available to you, since fork & threading bugs are 
+likely to be fixed there. 
+
 KNOWN BUGS
 
 Due to the way we handle signal traps, you cannot start more than one

data/Rakefile

@@ -1,70 +1,39 @@
-require 'rspec'
-require 'rspec/core/rake_task'
-Rspec::Core::RakeTask.new
-task :default => :spec
-
 require "rubygems"
-require "rake/gempackagetask"
-require "rake/rdoctask"
-
-# This builds the actual gem. For details of what all these options
-# mean, and other ones you can add, check the documentation here:
-#
-#   http://rubygems.org/read/chapter/20
-#
-spec = Gem::Specification.new do |s|
-
-  # Change these as appropriate
-  s.name              = "procrastinate"
-  s.version           = "0.3.1"
-  s.summary           = "Framework to run tasks in separate processes."
-  s.authors           = ['Kaspar Schiess', 'Patrick Marchi']
-  s.email             = ['kaspar.schiess@absurd.li', 'mail@patrickmarchi.ch']
-  s.homepage          = "http://github.com/kschiess/procrastinate"
-
-  s.has_rdoc          = true
-  s.extra_rdoc_files  = %w(README)
-  s.rdoc_options      = %w(--main README)
+require "rdoc/task"
+require 'rspec/core/rake_task'
+require 'rubygems/package_task'
 
-  # Add any extra files to include in the gem
-  s.files             = %w(LICENSE Rakefile README) + Dir.glob("{spec,lib/**/*}")
-  s.require_paths     = ["lib"]
+desc "Run all tests: Exhaustive."
+RSpec::Core::RakeTask.new
 
-  # If you want to depend on other gems, add them here, along with any
-  # relevant versions
-  s.add_dependency("state_machine", "~> 0.9.4")
+task :default => :spec
 
-  # If your tests use any gems, include them here
-  s.add_development_dependency("rspec")
-  s.add_development_dependency("flexmock")
+task :stats do
+  %w(lib spec).each do |path|
+    printf "%10s:", path
+    system %Q(find #{path} -name "*.rb" | xargs wc -l | grep total)
+  end
 end
 
-desc "Regenerate the .gemspec file for github/bundler."
-task :gemspec do
-  # Generate the gemspec file for github.
-  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
-  File.open(file, "w") {|f| f << spec.to_ruby }
+# Generate documentation
+RDoc::Task.new do |rdoc|
+  rdoc.title    = "procrastinate - a framework to run tasks in separate processes."
+  rdoc.options << '--line-numbers'
+  rdoc.options << '--fmt' << 'shtml' # explictly set shtml generator
+  rdoc.template = 'direct' # lighter template used on railsapi.com
+  rdoc.main = "README"
+  rdoc.rdoc_files.include("README", "lib/**/*.rb")
+  rdoc.rdoc_dir = "rdoc"
 end
 
-# This task actually builds the gem. We also regenerate a static
-# .gemspec file, which is useful if something (i.e. GitHub) will
-# be automatically building a gem for this project. If you're not
-# using GitHub, edit as appropriate.
-#
-# To publish your gem online, install the 'gemcutter' gem; Read more 
-# about that here: http://gemcutter.org/pages/gem_docs
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
-end
+desc 'Clear out RDoc'
+task :clean => [:clobber_rdoc, :clobber_package]
 
-# Generate documentation
-Rake::RDocTask.new do |rd|
-  rd.main = "README"
-  rd.rdoc_files.include("README", "lib/**/*.rb")
-  rd.rdoc_dir = "rdoc"
-end
+# This task actually builds the gem. 
+task :gem => :spec
+spec = eval(File.read('procrastinate.gemspec'))
 
-desc 'Clear out RDoc and generated packages'
-task :clean => [:clobber_rdoc, :clobber_package] do
-  rm "#{spec.name}.gemspec"
+desc "Generate the gem package."
+Gem::PackageTask.new(spec) do |pkg|
+  # pkg.need_tar = true
 end

data/examples/locking.rb

@@ -0,0 +1,26 @@
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'procrastinate'
+
+class Worker < Struct.new(:runtime)
+  def do_work
+    puts "#{Process.pid}: starting..."
+    runtime.lock('lock') do
+      puts "#{Process.pid}: holds lock"
+      sleep 0.1
+      puts "#{Process.pid}: releases"
+    end
+    puts "#{Process.pid}: done."
+  end
+end
+
+Procrastinate::Lock.base = '/tmp'   # This will have to be moved into scheduler
+scheduler = Procrastinate::Scheduler.start
+worker = scheduler.proxy(Worker.new(scheduler.runtime))
+
+10.times do 
+  worker.do_work
+end
+
+scheduler.shutdown

data/examples/pascal.rb

@@ -0,0 +1,30 @@
+
+# Computes Pascals triangle using procrastinate. This is mainly a stress test 
+# for result handling code and is NOT how I would parallelize this task!
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'procrastinate'
+require 'procrastinate/implicit'
+include Procrastinate
+
+V = Struct.new(:value) do
+  def ready?; true; end
+end
+
+current = [
+  V.new(1), 
+  V.new(1)
+]
+
+loop do
+  last = current
+
+  puts last.map { |e| sprintf("%3d", e.value) }.join(' ')
+  
+  current = [V.new(1)] + 
+    last.each_cons(2).map { |l,r| 
+      Procrastinate.schedule { l.value + r.value }
+    } + 
+    [V.new(1)]
+end

data/examples/simple.rb

@@ -0,0 +1,20 @@
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'procrastinate/implicit'
+
+class Worker
+  def do_work
+    puts "> Starting work in process #{Process.pid}"
+    sleep 2
+    puts "< Work completed in process #{Process.pid}"
+  end
+end
+
+worker = Procrastinate.proxy(Worker.new)
+
+10.times do 
+  worker.do_work
+end
+
+Procrastinate.join

data/examples/throttled.rb

@@ -0,0 +1,22 @@
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'procrastinate'
+include Procrastinate
+
+class Worker
+  def do_work
+    puts "> Starting work in process #{Process.pid}"
+    sleep 2
+    puts "< Work completed in process #{Process.pid}"
+  end
+end
+
+scheduler = Scheduler.start(SpawnStrategy::Throttled.new(5))
+worker = scheduler.proxy(Worker.new)
+
+10.times do 
+  worker.do_work
+end
+
+scheduler.shutdown

data/lib/procrastinate/implicit.rb

@@ -29,6 +29,17 @@ module Procrastinate
   end
   module_function :proxy
   
+  # call-seq: 
+  #   Procrastinate.schedule { some_work }
+  # 
+  # Schedules a block to be executed in its own thread. Returns the future that 
+  # will return the blocks return value. 
+  #
+  def schedule(&block)
+    scheduler.schedule(&block)
+  end
+  module_function :schedule
+  
   # call-seq: 
   #   Procrastinate.join
   #

data/lib/procrastinate/process_manager.rb

@@ -89,9 +89,12 @@ class Procrastinate::ProcessManager
     trap('CHLD', 'DEFAULT')
   end
     
-  # Called from the child management thread, will put that thread to sleep 
+  # Called from the child management thread, will put that thread to sleep
   # until someone requests it to become active again. See #wakeup. 
   #
+  # This method also depletes the child queue, reading end of processing
+  # messages from all childs and dispatching them to the children. 
+  #
   def wait_for_event
     cp_read_end = control_pipe.first
     
@@ -100,11 +103,10 @@ class Procrastinate::ProcessManager
       
       read_child_messages if ready.include? @cmc_server
 
-      # Kill children here, since we've just depleted the communication
-      # endpoint. This avoids the situation where the child process
-      # communicates but we remove it from our records before it can be told
-      # about it.
-      kill_children
+      # Send the tracking code for the child processes the final notifications
+      # and remove them from the children hash. At this point we know that
+      # no messages are waiting in the child queue.
+      finalize_children
       
       if ready.include? cp_read_end
         # Consume the data (not important)
@@ -112,18 +114,19 @@ class Procrastinate::ProcessManager
         return
       end
     end
-
-  # rescue Errno::EAGAIN, Errno::EINTR
-    # TODO Is this needed?
-    # A signal has been received. Mostly, this is as if we had received
-    # something in the control pipe.
   end
   
-  def kill_children
-    children.delete_if { |pid, child| child.dead? }
+  def finalize_children
+    children.
+      select { |pid, child| child.stopped? }.
+      each { |pid, child| child.finalize }
+
+    children.delete_if { |pid, child| 
+      child.removable? }
   end
 
-  # Once the @cmc_server endpoint is ready, loops and reads all child communication. 
+  # Once the @cmc_server endpoint is ready, loops and reads all child
+  # communication. 
   #
   def read_child_messages
     loop do
@@ -138,15 +141,18 @@ class Procrastinate::ProcessManager
   # that still needs decoding. 
   #
   def decode_and_handle_message(msg)
-    pid, obj = Marshal.load(msg)
+    pid, obj = begin
+      Marshal.load(msg)
+    rescue => b
+      # Messages that cannot be unmarshalled will be ignored. 
+      warn "Can't unmarshal child communication: #{b}"
+    end
+    
     if child=children[pid]
       child.incoming_message(obj)
     else
       warn "Communication from child #{pid} received, but child is gone."
     end
-  rescue => b
-    # Messages that cannot be unmarshalled will be ignored. 
-    warn "Can't unmarshal child communication."
   end
       
   # Calls completion handlers for all the childs that have now exited.
@@ -158,7 +164,11 @@ class Procrastinate::ProcessManager
 
       # Trigger the completion callback
       if child=children[child_pid]
-        child.died 
+        child.sigchld_received 
+        # Maybe there are messages queued for this child. If nothing is queued
+        # up, the thread will hang in the select in #wait_for_event unless
+        # we wake it up. 
+        wakeup
       end
     end
   rescue Errno::ECHILD
@@ -190,11 +200,16 @@ class Procrastinate::ProcessManager
 
       exit! # this seems to be needed to avoid rspecs cleanup tasks
     end
+        
+    # This should never fire: New children are spawned only after we loose 
+    # track of the old ones because they have been successfully processed.
+    fail "PID REUSE!" if children.has_key?(pid)
     
     # The spawning is done in the same thread as the reaping is done. This is 
     # why no race condition to the following line exists. (or in other code, 
     # for that matter.)
-    children[pid] = ChildProcess.new(completion_handler, result).tap { |s| s.start }
+    children[pid] = ChildProcess.new(completion_handler, result).
+      tap { |s| s.start }
   end
   
   # Gets executed in child process to clean up file handles and pipes that the

data/lib/procrastinate/process_manager/child_process.rb

@@ -2,21 +2,38 @@
 # a child exits and the object that will handle child-master communication
 # if desired.
 #
-class Procrastinate::ProcessManager::ChildProcess < Struct.new(:handler, :result, :state)
+Procrastinate::ProcessManager::ChildProcess = 
+  Struct.new(:handler, :result, :state) do
+  
+  def initialize(handler, result)
+    super(handler, result, "new")
+  end
+  
   state_machine :state, :initial => :new do
-    event(:start) { transition :new => :running }
-    event(:died)  { transition :running => :dead }
+    event(:start)             { transition :new => :running }
+    event(:sigchld_received)  { transition :running => :stopped }
+    event(:finalize)          { transition :stopped => :removable }
     
-    after_transition :on => :died, :do => :call_completion_handlers
+    after_transition :on => :sigchld_received, 
+      :do => :call_completion_handlers
+    after_transition :on => :finalize, 
+      :do => :notify_result
   end
   
-  # Calls the completion handler for the child. This is triggered by the
-  # transition into the 'dead' state. 
+  # Calls the completion handler for the child. At this stage, the process is
+  # not around anymore, but we still need to do some tracking. 
   #
   def call_completion_handlers
-    result.process_died if result
     handler.call if handler
   end
+  
+  # Notifies the childs result value that the child has died and that no
+  # more messages will be read. If this is the only notification, the result
+  # will yield a ChildDeath exception.
+  #
+  def notify_result
+    result.process_died if result
+  end
       
   # Handles incoming messages from the tasks process.
   #

data/lib/procrastinate/proxy.rb

@@ -17,7 +17,8 @@ class Procrastinate::Proxy
   
   def method_missing(name, *args, &block)
     if respond_to? name
-      task = Procrastinate::Task::MethodCall.new(@worker, name, args, block)
+      task = Procrastinate::Task::Callable.new(
+        lambda { @worker.send(name, *args, &block) })
       @scheduler.schedule(task)
       
       return task.result

data/lib/procrastinate/scheduler.rb

@@ -57,12 +57,22 @@ class Procrastinate::Scheduler
   # Called by the proxy to schedule work. You can implement your own Task
   # classes; the relevant interface consists of only a #run method. 
   #
-  def schedule(task)
+  def schedule(task=nil, &block)
     fail "Shutting down..." if @state != :running
+    
+    fail ArgumentError, "Either task or block must be given." \
+      if !task && !block
+    
+    if block
+      task = Procrastinate::Task::Callable.new(block)
+    end
+    
     task_queue << task
     
     # Create an occasion for spawning
     manager.wakeup
+    
+    task.result
   end
   
   # Waits for the currently queued work to complete. This can be used at the
@@ -90,14 +100,14 @@ class Procrastinate::Scheduler
   #
   def shutdown(hard=false)
     join unless hard
-    
+
     # Set the flag that will provoke shutdown
     @state = :real_shutdown
     # Wake the manager up, making it check the flag
     manager.wakeup
     # Wait for the manager to finish its work. This waits for child processes
     # and then reaps their result, avoiding zombies. 
-    @thread.join
+    @thread.join if @thread
   end
   
 private

data/lib/procrastinate/task.rb

@@ -2,5 +2,5 @@
 # A collection of tasks that can be performed with procrastinate. 
 #
 module Procrastinate::Task  
-  autoload :MethodCall, 'procrastinate/task/method_call'
+  autoload :Callable, 'procrastinate/task/callable'
 end

data/lib/procrastinate/task/callable.rb

@@ -0,0 +1,27 @@
+
+require 'procrastinate/task/result'
+
+module Procrastinate::Task
+  # Calls the block and returns the result of execution. 
+  #
+  class Callable
+    attr_reader :block
+    attr_reader :result
+    
+    def initialize(block)
+      @b = block
+      @result = Result.new
+    end
+  
+    # Runs this task. Gets passed an endpoint that can be used to communicate
+    # values back to the master. Every time you write a value to that endpoint
+    # (using #send), the server will call #incoming_message on the task object
+    # in the master process. This allows return values and other communication
+    # from children to the master (and to the caller in this case).
+    #
+    def run(endpoint)
+      r = @b.call
+      endpoint.send r if endpoint
+    end
+  end
+end

data/lib/procrastinate/task/result.rb

@@ -15,7 +15,8 @@ class Procrastinate::Task::Result
   #   *control thread* 
   #
   def incoming_message(obj)
-    return if ready?
+    fail "Read child answer too late, already marked dead." if ready? && @exception
+    fail "Double child answer." if ready?
     
     @value = obj
     @value_ready.set

data/lib/procrastinate/utils/one_time_flag.rb

@@ -1,3 +1,17 @@
+
+# A flag that will allow threads to wait until it is set. Once it is set, it
+# cannot be unset. 
+#
+# Guarantees that this class tries to make:
+# 
+# 1) No thread will start waiting for the flag once it is already set. There 
+#    no set-hole, meaning that no thread goes to sleep while we're waking up
+#    threads because the flag has been set. 
+#
+# Candidate stdlib classes violate some of these guarantees, here are some 
+# candidates: 
+#   * ConditionVariable - violates 1)
+#
 class Procrastinate::Utils::OneTimeFlag
   def initialize
     @waiting   = []
@@ -12,7 +26,9 @@ class Procrastinate::Utils::OneTimeFlag
     
     @waiting_m.synchronize do
       @waiting << Thread.current
-      @waiting_m.sleep(0.001) until set?
+      until set?
+        @waiting_m.sleep(0.001) 
+      end
     end
   end
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: procrastinate
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-13 00:00:00.000000000Z
+date: 2011-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: state_machine
-  requirement: &70138440330900 !ruby/object:Gem::Requirement
+  requirement: &70314600072220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 0.9.4
   type: :runtime
   prerelease: false
-  version_requirements: *70138440330900
+  version_requirements: *70314600072220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70138440330260 !ruby/object:Gem::Requirement
+  requirement: &70314600070260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70138440330260
+  version_requirements: *70314600070260
 - !ruby/object:Gem::Dependency
   name: flexmock
-  requirement: &70138440329160 !ruby/object:Gem::Requirement
+  requirement: &70314600069360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,7 +44,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70138440329160
+  version_requirements: *70314600069360
 description: 
 email:
 - kaspar.schiess@absurd.li
@@ -54,6 +54,7 @@ extensions: []
 extra_rdoc_files:
 - README
 files:
+- HISTORY.txt
 - LICENSE
 - Rakefile
 - README
@@ -71,13 +72,17 @@ files:
 - lib/procrastinate/spawn_strategy/simple.rb
 - lib/procrastinate/spawn_strategy/throttled.rb
 - lib/procrastinate/spawn_strategy.rb
-- lib/procrastinate/task/method_call.rb
+- lib/procrastinate/task/callable.rb
 - lib/procrastinate/task/result.rb
 - lib/procrastinate/task.rb
 - lib/procrastinate/utils/one_time_flag.rb
 - lib/procrastinate/utils/one_time_flag_ruby18_shim.rb
 - lib/procrastinate/utils.rb
 - lib/procrastinate.rb
+- examples/locking.rb
+- examples/pascal.rb
+- examples/simple.rb
+- examples/throttled.rb
 homepage: http://github.com/kschiess/procrastinate
 licenses: []
 post_install_message: 
@@ -92,6 +97,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 4532369798177350543
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/lib/procrastinate/task/method_call.rb

@@ -1,35 +0,0 @@
-
-require 'procrastinate/task/result'
-
-# Constructs an object of type +klass+ and calls a method on it. 
-#
-class Procrastinate::Task::MethodCall
-  include Procrastinate::Task
-  
-  attr_reader :i
-  attr_reader :m
-  attr_reader :a
-  attr_reader :b
-  
-  def initialize(instance, method, arguments, block)
-    @i = instance
-    @m = method
-    @a = arguments
-    @b = block
-  end
-  
-  # Runs this task. Gets passed an endpoint that can be used to communicate
-  # values back to the master. Every time you write a value to that endpoint
-  # (using #send), the server will call #incoming_message on the task object
-  # in the master process. This allows return values and other communication
-  # from children to the master (and to the caller in this case).
-  #
-  def run(endpoint)
-    r = @i.send(@m, *@a, &@b)
-    endpoint.send r if endpoint
-  end
-  
-  def result
-    @result ||= Result.new
-  end
-end