procrastinate 0.1.0

data/LICENSE

@@ -0,0 +1,23 @@
+
+ Copyright (c) 2010 Kaspar Schiess, Patrick Marchi
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,76 @@
+INTRO
+
+'procrastinate' does the process handling so you don't have to. It leaves you 
+to concentrate on what to run when, not orchestration of low level details. 
+
+This library will be ideal for quickly scheduling of a lot of long running
+tasks. You can easily control how many processes are run at any time. To get
+at/cron like scheduling, combine this library with 'rufus-scheduler'. 
+
+SYNOPSIS
+
+  require 'procrastinate'
+  include Procrastinate
+
+  class Worker
+    def do_work
+      puts "> Starting work in process #{Process.pid}"
+      sleep 2
+      puts "< Work completed in process #{Process.pid}"
+    end
+  end
+
+  scheduler = Scheduler.start(DispatchStrategy::Throttled.new(5))
+  worker = scheduler.create_proxy(Worker.new)
+
+  10.times do 
+    worker.do_work
+  end
+
+  scheduler.shutdown
+  
+The above example will output something like 
+
+  > Starting work in process 6651
+  > Starting work in process 6652
+  > Starting work in process 6653
+  > Starting work in process 6654
+  > Starting work in process 6655
+  < Work completed in process 6651
+  < Work completed in process 6652
+  < Work completed in process 6653
+  < Work completed in process 6654
+  < Work completed in process 6655
+  > Starting work in process 6659
+  > Starting work in process 6660
+  > Starting work in process 6656
+  > Starting work in process 6657
+  > Starting work in process 6658
+  < Work completed in process 6659
+  < Work completed in process 6660
+  < Work completed in process 6656
+  < Work completed in process 6657
+  < Work completed in process 6658
+
+COMPATIBILITY
+
+This library runs with at least Ruby 1.8.7 and Ruby 1.9. For running it with 
+Ruby 1.9, you need to patch it with the patch found at 
+
+  http://redmine.ruby-lang.org/issues/show/1525
+  
+or run it with at least r25844 of Ruby trunk.
+
+KNOWN BUGS
+
+Due to the way we handle signal traps, you cannot start more than one
+Scheduler. We might allow that in the future. 
+
+STATUS
+
+This code is not released as a gem yet. Once we iron out the bugs we'll 
+release it as 0.1.0. Alpha quality code. 
+
+Please see the LICENSE file for license information. 
+
+(c) 2010 Kaspar Schiess, Patrick Marchi

data/Rakefile

@@ -0,0 +1,69 @@
+require 'rspec'
+require 'rspec/core/rake_task'
+Rspec::Core::RakeTask.new
+task :default => :spec
+
+task :default => :spec
+
+require "rubygems"
+require "rake/gempackagetask"
+require "rake/rdoctask"
+
+# This builds the actual gem. For details of what all these options
+# mean, and other ones you can add, check the documentation here:
+#
+#   http://rubygems.org/read/chapter/20
+#
+spec = Gem::Specification.new do |s|
+
+  # Change these as appropriate
+  s.name              = "procrastinate"
+  s.version           = "0.1.0"
+  s.summary           = "Framework to run tasks in separate processes."
+  s.authors           = ['Kaspar Schiess', 'Patrick Marchi']
+  s.email             = ['kaspar.schiess@absurd.li', 'mail@patrickmarchi.ch']
+  s.homepage          = "http://github.com/kschiess/procrastinate"
+
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = %w(README)
+  s.rdoc_options      = %w(--main README)
+
+  # Add any extra files to include in the gem
+  s.files             = %w(LICENSE Rakefile README) + Dir.glob("{spec,lib/**/*}")
+  s.require_paths     = ["lib"]
+
+  # If you want to depend on other gems, add them here, along with any
+  # relevant versions
+  # s.add_dependency("blankslate", "~> 2.0")
+
+  # If your tests use any gems, include them here
+  s.add_development_dependency("rspec")
+  s.add_development_dependency("flexmock")
+end
+
+# This task actually builds the gem. We also regenerate a static
+# .gemspec file, which is useful if something (i.e. GitHub) will
+# be automatically building a gem for this project. If you're not
+# using GitHub, edit as appropriate.
+#
+# To publish your gem online, install the 'gemcutter' gem; Read more 
+# about that here: http://gemcutter.org/pages/gem_docs
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+
+  # Generate the gemspec file for github.
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
+# Generate documentation
+Rake::RDocTask.new do |rd|
+  rd.main = "README"
+  rd.rdoc_files.include("README", "lib/**/*.rb")
+  rd.rdoc_dir = "rdoc"
+end
+
+desc 'Clear out RDoc and generated packages'
+task :clean => [:clobber_rdoc, :clobber_package] do
+  rm "#{spec.name}.gemspec"
+end

data/lib/procrastinate.rb

@@ -0,0 +1,10 @@
+
+module Procrastinate; end
+
+require 'procrastinate/runtime'
+require 'procrastinate/lock'
+require 'procrastinate/dispatch_strategies'
+require 'procrastinate/tasks'
+require 'procrastinate/proxy'
+require 'procrastinate/dispatcher'
+require 'procrastinate/scheduler'

data/lib/procrastinate/dispatch_strategies.rb

@@ -0,0 +1,9 @@
+
+module Procrastinate::DispatchStrategy
+  # Raised when you request a shutdown and then schedule new work. 
+  #
+  class ShutdownRequested < StandardError; end
+end
+
+require 'procrastinate/dispatch_strategy/simple'
+require 'procrastinate/dispatch_strategy/throttled'

data/lib/procrastinate/dispatch_strategy/simple.rb

@@ -0,0 +1,47 @@
+
+require 'thread'
+
+class Procrastinate::DispatchStrategy::Simple
+  attr_reader :queue
+
+  # Client thread
+  def initialize
+    @queue = Queue.new
+    @shutdown_requested = false
+  end
+  
+  def shutdown_requested?
+    @shutdown_requested
+  end
+
+  # Client thread
+  def schedule(task)
+    raise ::ShutdownRequested if shutdown_requested?
+    
+    queue.push task
+  end
+  
+  # Dispatcher thread
+  def spawn_new_workers(dispatcher)
+    # Spawn tasks
+    spawn(dispatcher) while should_spawn?
+
+    # If the queue is empty now, maybe shutdown the dispatcher
+    dispatcher.request_stop if shutdown_requested? && queue.empty?
+  end
+  
+  
+  # Spawn a new task from the job queue. 
+  # Dispatcher thread
+  #
+  def spawn(dispatcher, &block)
+    dispatcher.spawn(queue.pop, &block)
+  end
+  def should_spawn?
+    not queue.empty?
+  end
+  
+  def shutdown
+    @shutdown_requested = true
+  end
+end

data/lib/procrastinate/dispatch_strategy/throttled.rb

@@ -0,0 +1,25 @@
+
+# A dispatcher strategy that throttles tasks starting and ensures that no
+# more than limit processes run concurrently. 
+#
+class Procrastinate::DispatchStrategy::Throttled < Procrastinate::DispatchStrategy::Simple
+  attr_reader :limit, :current
+  
+  # Client thread
+  def initialize(limit)
+    super()
+    
+    @limit = limit
+    @current = 0
+  end
+  
+  # Dispatcher thread
+  def spawn(dispatcher)
+    super(dispatcher) { @current -= 1 }
+    @current += 1
+  end
+  def should_spawn?
+    super &&
+      current < limit
+  end
+end

data/lib/procrastinate/dispatcher.rb

@@ -0,0 +1,164 @@
+
+# Dispatches and handles tasks and task completion. Only low level unixy
+# manipulation here, no strategy. The only method you should call from the
+# outside is #wakeup. 
+#
+class Procrastinate::Dispatcher
+  # The dispatcher runs in its own thread, which sleeps most of the time. 
+  attr_reader :thread
+  
+  # This pipe is used to wait for events in the master process. 
+  attr_reader :control_pipe
+  
+  # A hash of <pid, callback> that contains callbacks for all the child
+  # processes we spawn. Once the process is complete, the callback is called
+  # in the dispatcher/strategy's thread.
+  attr_reader :handlers
+  
+  # The strategy for dispatching new tasks. Makes all the decisions about
+  # when to launch what process.
+  #
+  attr_reader :strategy
+  
+  def initialize(strategy)
+    @strategy = strategy
+
+    @control_pipe = IO.pipe
+    @handlers = {}
+    @stop_requested = false
+  end
+
+  def self.start(strategy)
+    new(strategy).tap do |dispatcher| 
+      dispatcher.start
+    end
+  end
+  def start
+    register_signals
+    start_thread
+  end
+  
+  # Called from anywhere, will complete all running tasks and stop the
+  # dispatcher. 
+  #
+  def stop
+    request_stop
+    join
+    unregister_signals
+  end
+
+  # Called from the dispatcher thread, will cause the dispatcher to wait on
+  # all running tasks and then stop dispatching. 
+  #
+  def request_stop
+    @stop_requested = true
+    wakeup
+  end
+  
+  def stop_requested?
+    @stop_requested
+  end
+  
+  def register_signals
+    trap('CHLD') { wakeup }
+  end
+  def unregister_signals
+    trap('CHLD', 'DEFAULT')
+  end
+  
+  def start_thread
+    @thread = Thread.new do
+      Thread.current.abort_on_exception = true
+      
+      # Loop until someone requests a shutdown.
+      loop do
+        wait_for_event
+        reap_workers
+        
+        break if stop_requested?
+
+        strategy.spawn_new_workers(self)
+      end
+      
+      wait_for_all_childs
+    end
+  end
+  
+  def wait_for_event
+    # Returns array<ready_for_read, ..., ...>
+    IO.select([control_pipe.first], nil, nil)
+
+    # Consume the data (not important)
+    control_pipe.first.read_nonblock(1024)
+  rescue Errno::EAGAIN, Errno::EINTR
+  end
+  
+  # Wake up the dispatcher thread. 
+  #
+  def wakeup
+    control_pipe.last.write '.'
+  # rescue IOError
+    # Ignore:
+  end
+  
+  # Waits until the dispatcher completes its work. If you don't initiate a
+  # shutdown, this may be forever.
+  #
+  def join
+    @thread.join
+  end
+  
+  # Calls completion handlers for all the childs that have now exited.
+  #     
+  def reap_workers
+    loop do
+      child_pid, status = Process.waitpid2(-1, Process::WNOHANG)
+      break unless child_pid
+
+      # Trigger the completion callback
+      handler = handlers.delete(child_pid)
+      handler.call if handler
+    end
+  rescue Errno::ECHILD
+    # Ignored: Child status has been reaped by someone else
+  end
+  
+  # Spawns a process to work on +task+. If a block is given, it is called
+  # when the task completes.
+  #
+  # Example: 
+  # 
+  #   spawn(wi) { puts "Task is complete" }
+  #
+  def spawn(task, &completion_handler)
+    pid = fork do
+      cleanup
+
+      task.run
+
+      exit! # this seems to be needed to avoid rspecs cleanup tasks
+    end
+    
+    handlers[pid] = completion_handler
+  end
+  
+  # Gets executed in child process to clean up file handles and pipes that the
+  # master holds. 
+  #
+  def cleanup
+    # Children dont need the parents signal handler
+    trap(:CHLD, 'DEFAULT')
+    
+    # The child doesn't need the control pipe for now.
+    control_pipe.each { |io| io.close }
+  end
+
+  # Waits for all childs to complete. 
+  #
+  def wait_for_all_childs
+    until handlers.empty?
+      sleep 0.01
+      reap_workers
+    end
+  end
+end

data/lib/procrastinate/lock.rb

@@ -0,0 +1,34 @@
+
+# A file based lock below a base directory that is identified by Lock.base. 
+#
+class Procrastinate::Lock
+  class << self
+    # Base directory for all lock files that are created. 
+    #
+    attr_accessor :base
+  end
+  
+  attr_reader :name   # name of the lock
+  attr_reader :file   # file handle of the lock
+  def initialize(name)
+    @name = name
+    @file = File.open(
+      File.join(
+        self.class.base, name), 
+      'w+')
+  end
+  
+  def acquire
+    file.flock File::LOCK_EX
+  end
+  def release
+    file.flock File::LOCK_UN
+  end
+  
+  def synchronize
+    acquire
+    yield
+  ensure 
+    release
+  end
+end

data/lib/procrastinate/proxy.rb

@@ -0,0 +1,20 @@
+
+class Procrastinate::Proxy
+  def initialize(worker, scheduler)
+    @worker = worker
+    @scheduler = scheduler
+  end
+  
+  def respond_to?(name)
+    @worker.respond_to?(name)
+  end
+  
+  def method_missing(name, *args, &block)
+    if respond_to? name
+      @scheduler.schedule( 
+        Procrastinate::Task::MethodCall.new(@worker, name, args, block))
+    else
+      super
+    end
+  end
+end

data/lib/procrastinate/runtime.rb

@@ -0,0 +1,13 @@
+
+# An instance of this class obtained from the scheduler can be used to perform
+# synchronisation and other communication with the scheduler. 
+#
+class Procrastinate::Runtime
+  def lock(name)
+    lock = Procrastinate::Lock.new(name)
+    
+    lock.synchronize do
+      yield
+    end
+  end
+end

data/lib/procrastinate/scheduler.rb

@@ -0,0 +1,41 @@
+class Procrastinate::Scheduler
+  attr_reader :dispatcher
+  attr_reader :strategy
+  
+  def initialize
+    @shutdown_requested = false
+  end
+  
+  # Start a new scheduler
+  def self.start(strategy=nil)
+    new.start(strategy)
+  end
+  def start(strategy=nil)
+    @strategy   = strategy || Procrastinate::DispatchStrategy::Simple.new
+    @dispatcher = Procrastinate::Dispatcher.start(@strategy)
+    
+    self
+  end
+  
+  def create_proxy(worker)
+    return Procrastinate::Proxy.new(worker, self)
+  end
+  
+  # Returns a runtime linked to this scheduler. 
+  #
+  def runtime
+    Procrastinate::Runtime.new
+  end
+    
+  # Called by the proxy to schedule work.
+  #
+  def schedule(task)
+    strategy.schedule(task)
+    dispatcher.wakeup
+  end
+  
+  def shutdown
+    strategy.shutdown
+    dispatcher.join
+  end
+end

data/lib/procrastinate/tasks.rb

@@ -0,0 +1,16 @@
+module Procrastinate::Task
+  # Constructs an object of type +klass+ and calls a method on it. 
+  #
+  class MethodCall
+    def initialize(instance, method, arguments, block)
+      @instance = instance
+      @method = method
+      @arguments = arguments
+      @block = block
+    end
+    
+    def run
+      @instance.send(@method, *@arguments, &@block)
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification 
+name: procrastinate
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kaspar Schiess
+- Patrick Marchi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-10 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: flexmock
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: 
+email: 
+- kaspar.schiess@absurd.li
+- mail@patrickmarchi.ch
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- LICENSE
+- Rakefile
+- README
+- lib/procrastinate/dispatch_strategies.rb
+- lib/procrastinate/dispatch_strategy/simple.rb
+- lib/procrastinate/dispatch_strategy/throttled.rb
+- lib/procrastinate/dispatcher.rb
+- lib/procrastinate/lock.rb
+- lib/procrastinate/proxy.rb
+- lib/procrastinate/runtime.rb
+- lib/procrastinate/scheduler.rb
+- lib/procrastinate/tasks.rb
+- lib/procrastinate.rb
+has_rdoc: true
+homepage: http://github.com/kschiess/procrastinate
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Framework to run tasks in separate processes.
+test_files: []
+