procrastinate 0.1.0 → 0.2.0

data/README

@@ -20,7 +20,7 @@ SYNOPSIS
     end
   end
 
-  scheduler = Scheduler.start(DispatchStrategy::Throttled.new(5))
+  scheduler = Scheduler.start(SpawnStrategy::Throttled.new(5))
   worker = scheduler.create_proxy(Worker.new)
 
   10.times do 
@@ -54,22 +54,23 @@ The above example will output something like
 
 COMPATIBILITY
 
-This library runs with at least Ruby 1.8.7 and Ruby 1.9. For running it with 
-Ruby 1.9, you need to patch it with the patch found at 
-
-  http://redmine.ruby-lang.org/issues/show/1525
-  
-or run it with at least r25844 of Ruby trunk.
+This library runs with at least Ruby 1.8.7 and Ruby 1.9. Ruby 1.9 support
+might be spotty, because the threading primitives in Ruby 1.9 are still 
+somewhat buggy. 
 
 KNOWN BUGS
 
 Due to the way we handle signal traps, you cannot start more than one
 Scheduler. We might allow that in the future. 
 
+Also: signal traps interact with other libraries and might cause things to 
+break. This is the real world. 
+
 STATUS
 
-This code is not released as a gem yet. Once we iron out the bugs we'll 
-release it as 0.1.0. Alpha quality code. 
+We're still adding features that we believe must be in 1.0. What is there 
+mostly works; Multi-{Processing, Threading} is always a difficult topic and
+we're glad to receive bug reports.  
 
 Please see the LICENSE file for license information. 
 

data/Rakefile

@@ -3,8 +3,6 @@ require 'rspec/core/rake_task'
 Rspec::Core::RakeTask.new
 task :default => :spec
 
-task :default => :spec
-
 require "rubygems"
 require "rake/gempackagetask"
 require "rake/rdoctask"
@@ -18,7 +16,7 @@ spec = Gem::Specification.new do |s|
 
   # Change these as appropriate
   s.name              = "procrastinate"
-  s.version           = "0.1.0"
+  s.version           = "0.2.0"
   s.summary           = "Framework to run tasks in separate processes."
   s.authors           = ['Kaspar Schiess', 'Patrick Marchi']
   s.email             = ['kaspar.schiess@absurd.li', 'mail@patrickmarchi.ch']
@@ -34,13 +32,20 @@ spec = Gem::Specification.new do |s|
 
   # If you want to depend on other gems, add them here, along with any
   # relevant versions
-  # s.add_dependency("blankslate", "~> 2.0")
+  s.add_dependency("state_machine", "~> 0.9.4")
 
   # If your tests use any gems, include them here
   s.add_development_dependency("rspec")
   s.add_development_dependency("flexmock")
 end
 
+desc "Regenerate the .gemspec file for github/bundler."
+task :gemspec do
+  # Generate the gemspec file for github.
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
 # This task actually builds the gem. We also regenerate a static
 # .gemspec file, which is useful if something (i.e. GitHub) will
 # be automatically building a gem for this project. If you're not
@@ -50,10 +55,6 @@ end
 # about that here: http://gemcutter.org/pages/gem_docs
 Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
-
-  # Generate the gemspec file for github.
-  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
-  File.open(file, "w") {|f| f << spec.to_ruby }
 end
 
 # Generate documentation

data/lib/procrastinate/ipc/endpoint.rb

@@ -0,0 +1,106 @@
+
+# A communication endpoint. This acts as a factory and hub for the whole 
+# IPC library.
+#
+module Procrastinate::IPC::Endpoint
+  def anonymous
+    Anonymous.new
+  end
+  module_function :anonymous
+  
+  # Works the same as IO.select, only that it doesn't care about write and 
+  # error readiness, only read. You can mix IPC::Endpoints and normal IO
+  # instances freely. 
+  #
+  def select(read_array, timeout=nil)
+    # This maps real system IO instances to wrapper objects. Return the thing
+    # to the right if IO.select returns the thing to the left. 
+    mapping = Hash.new
+    waiting = []
+    
+    read_array.each { |io_or_endpoint| 
+      if io_or_endpoint.respond_to?(:select_ios)
+        waiting << io_or_endpoint if io_or_endpoint.waiting?
+        
+        io_or_endpoint.select_ios.each do |io|
+          mapping[io] = io_or_endpoint
+        end
+      else
+        mapping[io_or_endpoint] = io_or_endpoint
+      end
+    }
+    
+    return waiting unless waiting.empty?
+    
+    system_io = IO.select(mapping.keys, nil, nil, timeout)
+    if system_io
+      return system_io.first.
+        # Map returned selectors to their object counterparts and then only
+        # return once (if more than one was returned).
+        map { |e| mapping[e] }.uniq   
+    end
+  end
+  module_function :select
+  
+  class Anonymous
+    def initialize
+      @re, @we = IO.pipe
+    end
+    
+    def server
+      Server.new(@re)
+    end
+    def client
+      Client.new(@we)
+    end
+  end
+  
+  class Anonymous::Server
+    attr_reader :pipe
+    attr_reader :waiting
+    def initialize(pipe)
+      @pipe = pipe
+      @waiting = Array.new
+    end
+    
+    def receive(timeout=nil)
+      return waiting.shift if waiting?
+      
+      loop do
+        buffer = pipe.read_nonblock(1024*1024*1024)
+      
+        while buffer.size > 0
+          size = buffer.slice!(0...4).unpack('l').first
+          waiting << buffer.slice!(0...size)
+        end
+      
+        return waiting.shift if waiting?
+      end
+    end
+    
+    # True if there are queued messages in the Endpoint stack. If this is 
+    # false, a receive might block. 
+    #
+    def waiting?
+      not waiting.empty?
+    end
+    
+    # Return underlying IOs for select.
+    #
+    def select_ios
+      [@pipe]
+    end
+  end
+  
+  class Anonymous::Client
+    attr_reader :pipe
+    def initialize(pipe)
+      @pipe = pipe
+    end
+    
+    def send(msg)
+      buffer = [msg.size].pack('l') + msg
+      pipe.write(buffer)
+    end
+  end
+end

data/lib/procrastinate/ipc.rb

@@ -0,0 +1,6 @@
+
+# Holds classes for interprocess communication. 
+#
+module Procrastinate::IPC
+  autoload :Endpoint, 'procrastinate/ipc/endpoint'
+end

data/lib/procrastinate/process_manager.rb

@@ -0,0 +1,247 @@
+
+require 'state_machine'
+
+# Dispatches and handles tasks and task completion. Only low level unixy
+# manipulation here, no strategy. The only methods you should call from the
+# outside are #setup, #step, #wakeup and #shutdown. 
+#
+class Procrastinate::ProcessManager
+  include Procrastinate::IPC
+  
+  # This pipe is used to wait for events in the master process. 
+  attr_reader :control_pipe
+  
+  # A hash of <pid, callback> that contains callbacks for all the child
+  # processes we spawn. Once the process is complete, the callback is called
+  # in the procrastinate thread.
+  attr_reader :children
+  
+  # A class that acts as a filter between ProcessManager and the endpoint it
+  # uses to communicate with its children. This converts Ruby objects into
+  # Strings and also sends process id. 
+  #
+  class ObjectEndpoint < Struct.new(:endpoint, :pid)
+    def send(obj)
+      msg = Marshal.dump([pid, obj])
+      endpoint.send(msg)
+    end
+  end
+  
+  # A <completion handler, result> tuple that stores the handler to call when
+  # a child exits and the object that will handle child-master communication
+  # if desired.
+  #
+  class Child < Struct.new(:handler, :result, :state)
+    state_machine :state, :initial => :new do
+      event(:start) { transition :new => :running }
+      event(:died)  { transition :running => :dead }
+      
+      after_transition :on => :died, :do => :call_completion_handlers
+    end
+    
+    # Calls the completion handler for the child. This is triggered by the
+    # transition into the 'dead' state. 
+    #
+    def call_completion_handlers
+      result.process_died if result
+      handler.call if handler
+    end
+        
+    # Handles incoming messages from the tasks process.
+    #
+    def incoming_message(obj)
+      result.incoming_message(obj) if result
+    end
+  end
+  
+  def initialize
+    # This controls process manager wakeup
+    @control_pipe = IO.pipe
+    
+    # All presently running children
+    @children = {}
+    
+    # Child Master Communication (cmc)
+    endpoint = Endpoint.anonymous
+    @cmc_server = endpoint.server
+    @cmc_client = endpoint.client
+  end
+  
+  # Sets up resource usage for dispatcher. You must call this before dispatcher
+  # can start its work. 
+  #
+  def setup
+    register_signals
+  end
+  
+  # Performs one step in the dispatchers work. This will sleep and wait
+  # for work to be done, then wake up and reap processes that are still 
+  # pending. This method will mostly sleep. 
+  #
+  def step
+    # Sleep until either work arrives or we receive a SIGCHLD
+    wait_for_event
+    # Reap all processes that have terminated in the meantime.
+    reap_childs
+  end
+  
+  # Tears down the dispatcher. This frees resources that have been allocated
+  # and waits for all children to terminate. 
+  #
+  def teardown
+    wait_for_all_childs
+    unregister_signals
+  end
+  
+  # Wake up the dispatcher thread. 
+  #
+  def wakeup
+    control_pipe.last.write '.'
+  # rescue IOError
+    # Ignore:
+  end
+  
+  # Internal methods below this point. ---------------------------------------
+
+  # Register signals that aid in child care. NB: Because we do this globally, 
+  # holding more than one dispatcher in a process will not work yet. 
+  #
+  def register_signals
+    trap('CHLD') { wakeup }
+  end
+  
+  # Unregister signals. Process should be as before. 
+  #
+  def unregister_signals
+    trap('CHLD', 'DEFAULT')
+  end
+    
+  # Called from the child management thread, will put that thread to sleep 
+  # until someone requests it to become active again. See #wakeup. 
+  #
+  def wait_for_event
+    cp_read_end = control_pipe.first
+    
+    loop do # until we have input in the cp_read_end (control_pipe)
+      ready = Endpoint.select([cp_read_end, @cmc_server])
+      
+      read_child_messages if ready.include? @cmc_server
+
+      # Kill children here, since we've just depleted the communication
+      # endpoint. This avoids the situation where the child process
+      # communicates but we remove it from our records before it can be told
+      # about it.
+      kill_children
+      
+      if ready.include? cp_read_end
+        # Consume the data (not important)
+        cp_read_end.read_nonblock(1024)
+        return
+      end
+    end
+
+  # rescue Errno::EAGAIN, Errno::EINTR
+    # TODO Is this needed?
+    # A signal has been received. Mostly, this is as if we had received
+    # something in the control pipe.
+  end
+  
+  def kill_children
+    children.delete_if { |pid, child| child.dead? }
+  end
+
+  # Once the @cmc_server endpoint is ready, loops and reads all child communication. 
+  #
+  def read_child_messages
+    loop do
+      msg = @cmc_server.receive
+      decode_and_handle_message(msg)
+      
+      break unless @cmc_server.waiting?
+    end
+  end
+  
+  # Called for every message sent from a child. The +msg+ param here is a string
+  # that still needs decoding. 
+  #
+  def decode_and_handle_message(msg)
+    pid, obj = Marshal.load(msg)
+    if child=children[pid]
+      child.incoming_message(obj)
+    else
+      warn "Communication from child #{pid} received, but child is gone."
+    end
+  rescue => b
+    # Messages that cannot be unmarshalled will be ignored. 
+    warn "Can't unmarshal child communication."
+  end
+      
+  # Calls completion handlers for all the childs that have now exited.
+  #     
+  def reap_childs
+    loop do
+      child_pid, status = Process.waitpid(-1, Process::WNOHANG)
+      break unless child_pid
+
+      # Trigger the completion callback
+      if child=children[child_pid]
+        child.died 
+      end
+    end
+  rescue Errno::ECHILD
+    # Ignore: This means that no childs remain. 
+  end
+  
+  # Spawns a process to work on +task+. If a block is given, it is called
+  # when the task completes. This method should only be called from a strategy
+  # inside the dispatchers thread. Otherwise it will expose threading issues. 
+  #
+  # Example: 
+  # 
+  #   spawn(wi) { |pid| puts "Task is complete" }
+  #
+  def create_process(task, &completion_handler)
+    # Tasks that are interested in getting messages from their childs must 
+    # provide a result object that handles incoming 'result' messages.
+    result = task.result
+    
+    pid = fork do
+      cleanup
+            
+      if result
+        endpoint = ObjectEndpoint.new(@cmc_client, Process.pid)
+        task.run(endpoint)
+      else
+        task.run(nil)
+      end
+
+      exit! # this seems to be needed to avoid rspecs cleanup tasks
+    end
+    
+    # The spawning is done in the same thread as the reaping is done. This is 
+    # why no race condition to the following line exists. (or in other code, 
+    # for that matter.)
+    children[pid] = Child.new(completion_handler, result).tap { |s| s.start }
+  end
+  
+  # Gets executed in child process to clean up file handles and pipes that the
+  # master holds. 
+  #
+  def cleanup
+    # Children dont need the parents signal handler
+    unregister_signals
+    
+    # The child doesn't need the control pipe for now.
+    control_pipe.each { |io| io.close }
+  end
+
+  # Waits for all childs to complete. 
+  #
+  def wait_for_all_childs
+    # TODO Maybe signal KILL to children after some time. 
+    until children.all? { |p, c| c.dead? }
+      wait_for_event
+      reap_childs
+    end
+  end
+end

data/lib/procrastinate/proxy.rb

@@ -1,6 +1,12 @@
-
+# A proxy class that will translate all method calls made on it to method 
+# calls inside their own process via the Scheduler. 
+#
 class Procrastinate::Proxy
-  def initialize(worker, scheduler)
+  # Create a new proxy class. +worker+ is an instance of the class that we 
+  # want to perform work in, +scheduler+ is where the work will be scheduled. 
+  # Don't call this on your own, instead use Scheduler#create_proxy. 
+  #
+  def initialize(worker, scheduler) # :nodoc: 
     @worker = worker
     @scheduler = scheduler
   end
@@ -11,8 +17,10 @@ class Procrastinate::Proxy
   
   def method_missing(name, *args, &block)
     if respond_to? name
-      @scheduler.schedule( 
-        Procrastinate::Task::MethodCall.new(@worker, name, args, block))
+      task = Procrastinate::Task::MethodCall.new(@worker, name, args, block)
+      @scheduler.schedule(task)
+      
+      return task.result
     else
       super
     end

data/lib/procrastinate/scheduler.rb

@@ -1,41 +1,141 @@
+
+require 'thread'
+
+# API Frontend for the procrastinate library. Allows scheduling of tasks and
+# workers in seperate processes and provides minimal locking primitives. 
+#
+# Each scheduler owns its own thread that does all the processing. The
+# interface between your main thread and the procrastinate thread is defined
+# in this class.
+#
 class Procrastinate::Scheduler
-  attr_reader :dispatcher
+  attr_reader :manager
   attr_reader :strategy
-  
-  def initialize
-    @shutdown_requested = false
+  attr_reader :task_queue
+    
+  def initialize(strategy)
+    @strategy   = strategy || Procrastinate::SpawnStrategy::Simple.new
+    @manager = Procrastinate::ProcessManager.new
+
+    # State takes three values: :running, :soft_shutdown, :real_shutdown
+    # :soft_shutdown will not accept any new tasks and wait for completion
+    # :real_shutdown will stop as soon as possible (still closing down nicely)
+    @state      = :running
+    @task_queue = Queue.new
   end
   
-  # Start a new scheduler
+  # Starts a new scheduler
+  #
   def self.start(strategy=nil)
-    new.start(strategy)
+    new(strategy).
+      tap { |obj| obj.start }
   end
-  def start(strategy=nil)
-    @strategy   = strategy || Procrastinate::DispatchStrategy::Simple.new
-    @dispatcher = Procrastinate::Dispatcher.start(@strategy)
-    
-    self
+  def start
+    start_thread
   end
   
+  # Returns a proxy for the +worker+ instance that will allow executing its
+  # methods in a new process. 
+  #
+  # Example: 
+  #
+  #   proxy = scheduler.create_proxy(worker)
+  #   status = proxy.do_some_work    # will execute later and in its own process
+  #
   def create_proxy(worker)
     return Procrastinate::Proxy.new(worker, self)
   end
   
-  # Returns a runtime linked to this scheduler. 
+  # Returns a runtime linked to this scheduler. This method should only be
+  # used inside task execution processes; If you call it from your main 
+  # process, the result is undefined.
   #
   def runtime
     Procrastinate::Runtime.new
   end
     
-  # Called by the proxy to schedule work.
+  # Called by the proxy to schedule work. You can implement your own Task
+  # classes; the relevant interface consists of only a #run method. 
   #
   def schedule(task)
-    strategy.schedule(task)
-    dispatcher.wakeup
+    fail "Shutting down..." if @state != :running
+    task_queue << task
+    
+    # Create an occasion for spawning
+    manager.wakeup
+  end
+  
+  # Immediately shuts down the procrastinate thread and frees resources. 
+  # If there are any tasks left in the queue, they will NOT be executed. 
+  #
+  def shutdown(hard=false)
+    unless hard
+      @state = :soft_shutdown
+      loop do
+        manager.wakeup
+        break if task_queue.empty?
+      end
+    end
+    
+    # Set the flag that will provoke shutdown
+    @state = :real_shutdown
+    # Wake the manager up, making it check the flag
+    manager.wakeup
+    # Wait for the manager to finish its work. This waits for child processes
+    # and then reaps their result, avoiding zombies. 
+    @thread.join
   end
   
-  def shutdown
-    strategy.shutdown
-    dispatcher.join
+private
+  # Spawns new tasks (if needed). This is only ever called from the control
+  # thread (see below). 
+  # 
+  def spawn
+    while strategy.should_spawn? && !task_queue.empty?
+      task = task_queue.pop
+      manager.create_process(task) do
+        strategy.notify_dead
+      end
+      strategy.notify_spawn
+    end
+  end
+  
+  # This is the content of the control thread that is spawned with
+  # #start_thread
+  #
+  def run
+    # Start managers work
+    manager.setup
+
+    # Loop until someone requests a shutdown.
+    loop do
+      manager.step
+
+      break if @state == :real_shutdown
+      spawn
+    end
+
+    manager.teardown
+  rescue => ex
+    # Sometimes exceptions vanish silently. This will avoid that, even though
+    # they should abort the whole process.
+    
+    warn "Exception #{ex.inspect} caught."
+    ex.backtrace.first(5).each do |line|
+      warn line
+    end
+    
+    raise
+  end
+
+  # Hosts the control thread that runs in parallel with your code. This thread
+  # handles child spawning and reaping. 
+  #
+  def start_thread # :nodoc: 
+    Thread.abort_on_exception = true
+    
+    @thread = Thread.new do
+      run
+    end
   end
 end

data/lib/procrastinate/spawn_strategy/simple.rb

@@ -0,0 +1,12 @@
+
+class Procrastinate::SpawnStrategy::Simple
+  def should_spawn?
+    true
+  end
+  
+  def notify_spawn
+  end
+  
+  def notify_dead
+  end
+end

data/lib/procrastinate/{dispatch_strategy → spawn_strategy}/throttled.rb

@@ -2,7 +2,7 @@
 # A dispatcher strategy that throttles tasks starting and ensures that no
 # more than limit processes run concurrently. 
 #
-class Procrastinate::DispatchStrategy::Throttled < Procrastinate::DispatchStrategy::Simple
+class Procrastinate::SpawnStrategy::Throttled < Procrastinate::SpawnStrategy::Simple
   attr_reader :limit, :current
   
   # Client thread
@@ -13,13 +13,17 @@ class Procrastinate::DispatchStrategy::Throttled < Procrastinate::DispatchStrate
     @current = 0
   end
   
-  # Dispatcher thread
-  def spawn(dispatcher)
-    super(dispatcher) { @current -= 1 }
+  def should_spawn?
+    current < limit
+  end
+  
+  def notify_spawn
     @current += 1
+    warn "Throttled reports too many births!" if current > limit
   end
-  def should_spawn?
-    super &&
-      current < limit
+  
+  def notify_dead
+    @current -= 1
+    warn "Throttled reports more deaths than births?!" if current < 0
   end
 end

data/lib/procrastinate/spawn_strategy.rb

@@ -0,0 +1,5 @@
+
+module Procrastinate::SpawnStrategy
+  autoload :Simple, 'procrastinate/spawn_strategy/simple'
+  autoload :Throttled, 'procrastinate/spawn_strategy/throttled'
+end

data/lib/procrastinate/task/method_call.rb

@@ -0,0 +1,35 @@
+
+require 'procrastinate/task/result'
+
+# Constructs an object of type +klass+ and calls a method on it. 
+#
+class Procrastinate::Task::MethodCall
+  include Procrastinate::Task
+  
+  attr_reader :i
+  attr_reader :m
+  attr_reader :a
+  attr_reader :b
+  
+  def initialize(instance, method, arguments, block)
+    @i = instance
+    @m = method
+    @a = arguments
+    @b = block
+  end
+  
+  # Runs this task. Gets passed an endpoint that can be used to communicate
+  # values back to the master. Every time you write a value to that endpoint
+  # (using #send), the server will call #incoming_message on the task object
+  # in the master process. This allows return values and other communication
+  # from children to the master (and to the caller in this case).
+  #
+  def run(endpoint)
+    r = @i.send(@m, *@a, &@b)
+    endpoint.send r if endpoint
+  end
+  
+  def result
+    @result ||= Result.new
+  end
+end

data/lib/procrastinate/task/result.rb

@@ -0,0 +1,47 @@
+
+require 'procrastinate/utils'
+
+# A single value result, like from a normal method call. Return an instance of
+# this from your task#result method to enable result handling. 
+#
+class Procrastinate::Task::Result
+  def initialize
+    @value_ready    = Procrastinate::Utils::OneTimeFlag.new
+    @value          = nil
+    @exception      = false
+  end
+  
+  # Gets passed all messages sent by the child process for this task.
+  #
+  def incoming_message(obj)
+    return if ready?
+    
+    @value = obj
+    @value_ready.set
+  end
+  
+  # Notifies this result that the process has died. If this happens before
+  # a process result is passed to #incoming_message, that message will never
+  # arrive. 
+  #
+  def process_died
+    return if ready?
+    
+    @exception = true    
+    @value_ready.set
+  end
+
+  def value
+    @value_ready.wait
+    
+    if @exception
+      raise Procrastinate::ChildDeath, "Child process died before producing a value."
+    else
+      @value
+    end
+  end
+  
+  def ready?
+    @value_ready.set?
+  end
+end

data/lib/procrastinate/task.rb

@@ -0,0 +1,6 @@
+
+# A collection of tasks that can be performed with procrastinate. 
+#
+module Procrastinate::Task  
+  autoload :MethodCall, 'procrastinate/task/method_call'
+end

data/lib/procrastinate/utils/one_time_flag.rb

@@ -0,0 +1,39 @@
+class Procrastinate::Utils::OneTimeFlag
+  def initialize
+    @waiting   = []
+    @waiting_m = Mutex.new
+    @set       = false
+  end
+  
+  # If the flag is set, does nothing. If it isn't, it blocks until the flag
+  # is set. 
+  def wait
+    return if set?
+    
+    @waiting_m.synchronize do
+      @waiting << Thread.current
+      @waiting_m.sleep(0.001) until set?
+    end
+  end
+  
+  # Sets the flag and releases all waiting threads.
+  #
+  def set
+    @set = true
+    @waiting_m.synchronize do
+      @waiting.each { |t| t.run }
+      @waiting = [] # cleanup
+    end
+  end
+  
+  # Non blocking: Is the flag set?
+  #
+  def set?
+    @set
+  end
+end
+
+if RUBY_VERSION =~ /^1.8/
+  require 'procrastinate/utils/one_time_flag_ruby18_shim'
+end
+

data/lib/procrastinate/utils/one_time_flag_ruby18_shim.rb

@@ -0,0 +1,32 @@
+class Procrastinate::Utils::OneTimeFlag
+  def initialize
+    @waiting_m  = Mutex.new
+    @waiting_cv = ConditionVariable.new
+    @set        = false
+  end
+  
+  # If the flag is set, does nothing. If it isn't, it blocks until the flag
+  # is set. 
+  def wait
+    return if set?
+    
+    @waiting_m.synchronize do
+      @waiting_cv.wait(@waiting_m)
+    end
+  end
+  
+  # Sets the flag and releases all waiting threads.
+  #
+  def set
+    @set = true
+    @waiting_m.synchronize do
+      @waiting_cv.broadcast
+    end
+  end
+  
+  # Non blocking: Is the flag set?
+  #
+  def set?
+    @set
+  end
+end

data/lib/procrastinate/utils.rb

@@ -0,0 +1,5 @@
+
+module Procrastinate::Utils
+end
+
+require 'procrastinate/utils/one_time_flag'

data/lib/procrastinate.rb

@@ -1,10 +1,17 @@
 
-module Procrastinate; end
+module Procrastinate
+  # Raised when you try to access a future value that belongs to a process
+  # that died before producing a value. 
+  #
+  class ChildDeath < StandardError; end
+  
+  autoload :Lock,     'procrastinate/lock'
+  autoload :Runtime,  'procrastinate/runtime'
+  autoload :IPC,      'procrastinate/ipc'
+  autoload :Task,     'procrastinate/task'
+end
 
-require 'procrastinate/runtime'
-require 'procrastinate/lock'
-require 'procrastinate/dispatch_strategies'
-require 'procrastinate/tasks'
+require 'procrastinate/spawn_strategy'
 require 'procrastinate/proxy'
-require 'procrastinate/dispatcher'
+require 'procrastinate/process_manager'
 require 'procrastinate/scheduler'

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Kaspar Schiess
@@ -15,13 +15,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-10 00:00:00 +01:00
+date: 2010-12-22 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: state_machine
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 4
+        version: 0.9.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -30,11 +45,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id001
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: flexmock
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -43,7 +58,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 description: 
 email: 
 - kaspar.schiess@absurd.li
@@ -58,15 +73,22 @@ files:
 - LICENSE
 - Rakefile
 - README
-- lib/procrastinate/dispatch_strategies.rb
-- lib/procrastinate/dispatch_strategy/simple.rb
-- lib/procrastinate/dispatch_strategy/throttled.rb
-- lib/procrastinate/dispatcher.rb
+- lib/procrastinate/ipc/endpoint.rb
+- lib/procrastinate/ipc.rb
 - lib/procrastinate/lock.rb
+- lib/procrastinate/process_manager.rb
 - lib/procrastinate/proxy.rb
 - lib/procrastinate/runtime.rb
 - lib/procrastinate/scheduler.rb
-- lib/procrastinate/tasks.rb
+- lib/procrastinate/spawn_strategy/simple.rb
+- lib/procrastinate/spawn_strategy/throttled.rb
+- lib/procrastinate/spawn_strategy.rb
+- lib/procrastinate/task/method_call.rb
+- lib/procrastinate/task/result.rb
+- lib/procrastinate/task.rb
+- lib/procrastinate/utils/one_time_flag.rb
+- lib/procrastinate/utils/one_time_flag_ruby18_shim.rb
+- lib/procrastinate/utils.rb
 - lib/procrastinate.rb
 has_rdoc: true
 homepage: http://github.com/kschiess/procrastinate

data/lib/procrastinate/dispatch_strategies.rb

@@ -1,9 +0,0 @@
-
-module Procrastinate::DispatchStrategy
-  # Raised when you request a shutdown and then schedule new work. 
-  #
-  class ShutdownRequested < StandardError; end
-end
-
-require 'procrastinate/dispatch_strategy/simple'
-require 'procrastinate/dispatch_strategy/throttled'

data/lib/procrastinate/dispatch_strategy/simple.rb

@@ -1,47 +0,0 @@
-
-require 'thread'
-
-class Procrastinate::DispatchStrategy::Simple
-  attr_reader :queue
-
-  # Client thread
-  def initialize
-    @queue = Queue.new
-    @shutdown_requested = false
-  end
-  
-  def shutdown_requested?
-    @shutdown_requested
-  end
-
-  # Client thread
-  def schedule(task)
-    raise ::ShutdownRequested if shutdown_requested?
-    
-    queue.push task
-  end
-  
-  # Dispatcher thread
-  def spawn_new_workers(dispatcher)
-    # Spawn tasks
-    spawn(dispatcher) while should_spawn?
-
-    # If the queue is empty now, maybe shutdown the dispatcher
-    dispatcher.request_stop if shutdown_requested? && queue.empty?
-  end
-  
-  
-  # Spawn a new task from the job queue. 
-  # Dispatcher thread
-  #
-  def spawn(dispatcher, &block)
-    dispatcher.spawn(queue.pop, &block)
-  end
-  def should_spawn?
-    not queue.empty?
-  end
-  
-  def shutdown
-    @shutdown_requested = true
-  end
-end

data/lib/procrastinate/dispatcher.rb

@@ -1,164 +0,0 @@
-
-# Dispatches and handles tasks and task completion. Only low level unixy
-# manipulation here, no strategy. The only method you should call from the
-# outside is #wakeup. 
-#
-class Procrastinate::Dispatcher
-  # The dispatcher runs in its own thread, which sleeps most of the time. 
-  attr_reader :thread
-  
-  # This pipe is used to wait for events in the master process. 
-  attr_reader :control_pipe
-  
-  # A hash of <pid, callback> that contains callbacks for all the child
-  # processes we spawn. Once the process is complete, the callback is called
-  # in the dispatcher/strategy's thread.
-  attr_reader :handlers
-  
-  # The strategy for dispatching new tasks. Makes all the decisions about
-  # when to launch what process.
-  #
-  attr_reader :strategy
-  
-  def initialize(strategy)
-    @strategy = strategy
-
-    @control_pipe = IO.pipe
-    @handlers = {}
-    @stop_requested = false
-  end
-
-  def self.start(strategy)
-    new(strategy).tap do |dispatcher| 
-      dispatcher.start
-    end
-  end
-  def start
-    register_signals
-    start_thread
-  end
-  
-  # Called from anywhere, will complete all running tasks and stop the
-  # dispatcher. 
-  #
-  def stop
-    request_stop
-    join
-    unregister_signals
-  end
-
-  # Called from the dispatcher thread, will cause the dispatcher to wait on
-  # all running tasks and then stop dispatching. 
-  #
-  def request_stop
-    @stop_requested = true
-    wakeup
-  end
-  
-  def stop_requested?
-    @stop_requested
-  end
-  
-  def register_signals
-    trap('CHLD') { wakeup }
-  end
-  def unregister_signals
-    trap('CHLD', 'DEFAULT')
-  end
-  
-  def start_thread
-    @thread = Thread.new do
-      Thread.current.abort_on_exception = true
-      
-      # Loop until someone requests a shutdown.
-      loop do
-        wait_for_event
-        reap_workers
-        
-        break if stop_requested?
-
-        strategy.spawn_new_workers(self)
-      end
-      
-      wait_for_all_childs
-    end
-  end
-  
-  def wait_for_event
-    # Returns array<ready_for_read, ..., ...>
-    IO.select([control_pipe.first], nil, nil)
-
-    # Consume the data (not important)
-    control_pipe.first.read_nonblock(1024)
-  rescue Errno::EAGAIN, Errno::EINTR
-  end
-  
-  # Wake up the dispatcher thread. 
-  #
-  def wakeup
-    control_pipe.last.write '.'
-  # rescue IOError
-    # Ignore:
-  end
-  
-  # Waits until the dispatcher completes its work. If you don't initiate a
-  # shutdown, this may be forever.
-  #
-  def join
-    @thread.join
-  end
-  
-  # Calls completion handlers for all the childs that have now exited.
-  #     
-  def reap_workers
-    loop do
-      child_pid, status = Process.waitpid2(-1, Process::WNOHANG)
-      break unless child_pid
-
-      # Trigger the completion callback
-      handler = handlers.delete(child_pid)
-      handler.call if handler
-    end
-  rescue Errno::ECHILD
-    # Ignored: Child status has been reaped by someone else
-  end
-  
-  # Spawns a process to work on +task+. If a block is given, it is called
-  # when the task completes.
-  #
-  # Example: 
-  # 
-  #   spawn(wi) { puts "Task is complete" }
-  #
-  def spawn(task, &completion_handler)
-    pid = fork do
-      cleanup
-
-      task.run
-
-      exit! # this seems to be needed to avoid rspecs cleanup tasks
-    end
-    
-    handlers[pid] = completion_handler
-  end
-  
-  # Gets executed in child process to clean up file handles and pipes that the
-  # master holds. 
-  #
-  def cleanup
-    # Children dont need the parents signal handler
-    trap(:CHLD, 'DEFAULT')
-    
-    # The child doesn't need the control pipe for now.
-    control_pipe.each { |io| io.close }
-  end
-
-  # Waits for all childs to complete. 
-  #
-  def wait_for_all_childs
-    until handlers.empty?
-      sleep 0.01
-      reap_workers
-    end
-  end
-end

data/lib/procrastinate/tasks.rb

@@ -1,16 +0,0 @@
-module Procrastinate::Task
-  # Constructs an object of type +klass+ and calls a method on it. 
-  #
-  class MethodCall
-    def initialize(instance, method, arguments, block)
-      @instance = instance
-      @method = method
-      @arguments = arguments
-      @block = block
-    end
-    
-    def run
-      @instance.send(@method, *@arguments, &@block)
-    end
-  end
-end