sisyphus 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1a96cdf599669e601888f70c06de7ae280b86fc6
-  data.tar.gz: 601d5544f93601e4b9aac5df9dad122040742300
+SHA256:
+  metadata.gz: 5b0d7aeac8761a528b7d084dcf254c284d541f1aabf3c728e35e5f25233fa8fe
+  data.tar.gz: 2358bceac26513eeab3f712138bbcd98cfda0fa9e3f3d38a1cdbbabacb762487
 SHA512:
-  metadata.gz: 731a337d88e4ca319a824c2fdb44f2343808adfd1a15762bfb6a2417666c7aca8182b2859bf495c33f4c48c170091061b36f18fb0088281258288fe677a5d55a
-  data.tar.gz: e4c2bcbfb26349558d25d705fb69f370cc05e4040f8f46bc4430794b448b1e621bcece6988b68318a2ee6dba128594001cd42f2212bf52ac9ec79125e9704afc
+  metadata.gz: 66317b7e08c7c1de9125e1300811e808641fc76dc29230bcdff93ec66e1a1a1f8841b4565cdbd54709517501f283b81dbcb5e1ba4ceaeefe5f5ccfa7566437bd
+  data.tar.gz: cc2fb6929498e3c6ab44eddbd6aa7439cfef3357fdff42e11998f25941c2c44a522b6a7375800fa35068ddd2964bf2869e0f81b01cdafc08ed2546088299f327

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sisyphus (0.2.2)
+    sisyphus (0.2.3)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -32,6 +32,10 @@ Getting started
      master or workers encounter. The logger should quack like a
      `Logger` instance from the Ruby stdlib. Exceptions are logged with the
      `Logger::WARN` level.
+   * `:execution_strategy` which can either be `Sisyphus::SimpleExecutionStrategy` or
+     `Sisyphus::ForkingExecutionStrategy`. This is the strategy used by
+     workers when performing the job. The default is
+     `Sisyphus::ForkingExecutionStrategy`.
 4. You can start workers by doing one of the following things:
    * Send the `start` message to the master, if the `options` hash was
      provided. This starts a run loop which monitors workers and
@@ -63,8 +67,6 @@ Things missing
 Sisyphus is still very much in its infancy, though the ambition isn't to build a [Resque] [resque] clone, but
 instead build as small a tool with as few features as possible.
 
-[resque]: https://github.com/resque/resque
-
 There are, however, still features that are missing:
 
 - Force killing workers
@@ -76,6 +78,7 @@ There are, however, still features that are missing:
 - Some sort of reaping of worker processes
 - Documentation
 
+[resque]: https://github.com/resque/resque
 [unicorn]: http://unicorn.bogomips.org/
 
 Contributing

data/Rakefile

@@ -5,8 +5,10 @@ task :sleeper do
   gem 'sisyphus'
   require 'sisyphus'
   require 'sisyphus/sleep'
+  require 'logger'
 
+  logger = Logger.new(STDOUT)
   job = Sisyphus::Sleep.new
-  master = Sisyphus::Master.new job, workers: 2
+  master = Sisyphus::Master.new job, workers: 2, logger: logger
   master.start
 end

data/lib/sisyphus/forking_execution_strategy.rb

@@ -0,0 +1,52 @@
+module Sisyphus
+  class ForkingExecutionStrategy
+
+    def execute(job, error_handler = ->(process_name, error) {})
+      if @child_pid = fork
+        ChildProcess.new(@child_pid).success?
+      else
+        perform job, error_handler
+      end
+    end
+
+    private
+
+    def perform(job, error_handler)
+      self.process_name = "Child of worker #{::Process.ppid}"
+      begin
+        job.perform
+        exit! 0
+      rescue ::Exception => e
+        error_handler.call process_name, e
+        exit! 1
+      end
+    end
+
+    def process_name
+      $0
+    end
+
+    def process_name=(name)
+      $0 = name
+    end
+
+    class ChildProcess
+      attr_reader :pid
+
+      def initialize(pid)
+        @pid = pid
+      end
+
+      def success?
+        status.success?
+      end
+
+      def status
+        _, status = ::Process.waitpid2 pid
+        status
+      end
+
+    end
+
+  end
+end

data/lib/sisyphus/master.rb

@@ -1,6 +1,8 @@
 require 'timeout'
 require_relative './worker'
+require_relative './forking_execution_strategy'
 require_relative './null_logger'
+require_relative './worker_pool'
 
 module Sisyphus
   class Master
@@ -8,12 +10,15 @@ module Sisyphus
 
     HANDLED_SIGNALS = [:INT, :TTIN, :TTOU]
 
-    attr_reader :logger
+    attr_reader :logger, :job, :number_of_workers, :execution_strategy
 
     def initialize(job, options = {})
-      @number_of_workers = options.fetch :workers, 0
-      @logger = options.fetch :logger, NullLogger.new
-      @workers = []
+      self.number_of_workers = options.fetch :workers, 0
+      @logger = options.fetch(:logger) { NullLogger.new }
+      @execution_strategy = options.fetch(:execution_strategy) { ForkingExecutionStrategy.new }
+
+      @worker_pool = options.fetch(:worker_pool) { WorkerPool.new self }
+
       @job = job
 
       self_reader, self_writer = IO.pipe
@@ -24,60 +29,54 @@ module Sisyphus
 
     def start
       trap_signals
-      @number_of_workers.times do
-        start_worker
+      number_of_workers.times do
+        @worker_pool.spawn_worker
         sleep rand(1000).fdiv(1000)
       end
       puts "Sisyphus::Master started with PID: #{Process.pid}"
       watch_for_output
     end
 
-    def start_worker
-      reader, writer = IO.pipe
-      if wpid = fork
-        writer.close
-        @workers << { pid: wpid, reader: reader }
-      else
-        reader.close
-        self.process_name = "Worker #{Process.pid}"
-        begin
-          worker = Worker.new(@job, writer, logger)
-          worker.setup
-          worker.start
-        rescue Exception => e
-          writer.write Worker::UNCAUGHT_ERROR
-          logger.warn(process_name) { e }
-          exit! 0
-        end
-      end
+    def create_worker
+      Worker.new(job, execution_strategy, logger)
     end
 
     def stop_worker(wpid)
-      if @workers.find { |w| w.fetch(:pid) == wpid }
+      if workers.find { |w| w.fetch(:pid) == wpid }
         Process.kill 'INT', wpid rescue Errno::ESRCH # Ignore if the process is already gone
       end
     end
 
     def stop_all
-      @workers.each do |worker|
+      workers.each do |worker|
         stop_worker worker.fetch(:pid)
       end
-      Timeout.timeout(30) do
-        watch_for_shutdown while worker_count > 0
+      begin
+        Timeout.timeout(30) do
+          watch_for_shutdown while worker_count > 0
+        end
+      rescue Timeout::Error => e
+        p "Timeout reached:", e
       end
     end
 
     def worker_count
-      @workers.length
+      workers.length
     end
 
     private
 
+    attr_writer :number_of_workers
+
+    def workers
+      @worker_pool.workers
+    end
+
     def watch_for_shutdown
       wpid, _ = Process.wait2
-      worker = @workers.find { |w| w.fetch(:pid) == wpid }
+      worker = workers.find { |w| w.fetch(:pid) == wpid }
       worker.fetch(:reader).close
-      @workers.delete worker
+      workers.delete worker
       wpid
     rescue Errno::ECHILD
     end
@@ -93,7 +92,7 @@ module Sisyphus
     end
 
     def process_signal_queue
-      handle_signal(Thread.main[:signal_queue].shift) until Thread.main[:signal_queue].empty?
+      handle_signal(signal_queue.shift) until signal_queue.empty?
     end
 
     def process_pipes(pipes)
@@ -107,26 +106,26 @@ module Sisyphus
 
     def process_output(pipes)
       pipes.each do |pipe|
-        restart_worker worker_pid(pipe) unless stopping?
+        respawn_worker worker_pid(pipe) unless stopping?
       end
     end
 
-    def restart_worker(wpid)
-      start_worker
+    def respawn_worker(wpid)
+      @worker_pool.spawn_worker
       stop_worker wpid
       watch_for_shutdown
     end
 
     def worker_pipes
       if worker_count > 0
-        @workers.map { |w| w.fetch(:reader) }
+        workers.map { |w| w.fetch(:reader) }
       else
         []
       end
     end
 
     def worker_pid(reader)
-      if worker = @workers.find { |w| w.fetch(:reader).fileno == reader.fileno }
+      if worker = workers.find { |w| w.fetch(:reader).fileno == reader.fileno }
         worker.fetch(:pid)
       else
         raise 'Unknown worker pipe'
@@ -142,7 +141,7 @@ module Sisyphus
     end
 
     def queue_signal(signal)
-      Thread.main[:signal_queue] << signal
+      signal_queue << signal
       @selfpipe[:writer].write_nonblock('.')
     rescue Errno::EAGAIN
       # Ignore
@@ -171,14 +170,14 @@ module Sisyphus
     end
 
     def handle_ttin
-      @number_of_workers += 1
-      start_worker
+      self.number_of_workers += 1
+      @worker_pool.spawn_worker
     end
 
     def handle_ttou
-      if @number_of_workers > 0
-        @number_of_workers -= 1
-        stop_worker(@workers.first.fetch(:pid))
+      if number_of_workers > 0
+        self.number_of_workers -= 1
+        stop_worker(workers.first.fetch(:pid))
       end
     end
 
@@ -190,12 +189,8 @@ module Sisyphus
       @stopping
     end
 
-    def process_name=(name)
-      $0 = name
-    end
-
-    def process_name
-      $0
+    def signal_queue
+      Thread.main[:signal_queue]
     end
   end
 end

data/lib/sisyphus/simple_execution_strategy.rb

@@ -0,0 +1,17 @@
+module Sisyphus
+  class SimpleExecutionStrategy
+
+    def execute(job, error_handler = ->(name, error) {})
+      job.perform
+    rescue Exception => e
+      error_handler.call process_name, e
+    end
+
+    private
+
+    def process_name
+      $0
+    end
+
+  end
+end

data/lib/sisyphus/sleep.rb

@@ -3,7 +3,7 @@ require_relative './job'
 module Sisyphus
   class Sleep < Job
     def perform
-      sleep 2
+      sleep 40
       puts "Goodmorning from #{Process.pid}"
       raise "Hest" if rand(10) % 2 == 0
     end

data/lib/sisyphus/worker.rb

@@ -2,49 +2,74 @@ module Sisyphus
   class Worker
     UNCAUGHT_ERROR = '.'
 
-    attr_reader :logger
+    attr_reader :logger, :execution_strategy, :job, :output, :to_master
 
-    def initialize(job, output, logger)
+    def initialize(job, execution_strategy, logger)
       @job = job
-      @output = output
+      @to_master, @output = IO.pipe
+      @execution_strategy = execution_strategy
       @logger = logger
+      @set_up = false
     end
 
-    def setup
-      @job.setup if @job.respond_to? :setup
+    def start
+      setup
+      run
     end
 
-    def start
+    def setup
       trap_signals
+      job.setup if job.respond_to? :setup
+      setup_done
+    rescue Exception => e
+      error_handler.call "Setup", e
+    end
 
+    def run
       loop do
         break if stopped?
         perform_job
-      end
+      end if set_up?
 
       exit! 0
     end
 
-    private
-
     def perform_job
-      if child = fork
-        _, status = Process.waitpid2 child
+      execution_strategy.execute job, error_handler
+    end
+
+    def error_handler
+      -> (name, error) {
+        return if stopped?
         begin
-          @output.write UNCAUGHT_ERROR unless status.success? || stopped?
+          logger.warn(name) { error }
+          output.write UNCAUGHT_ERROR
         rescue Errno::EAGAIN, Errno::EINTR
           # Ignore
         end
-      else
-        self.process_name = "Child of worker #{Process.ppid}"
-        begin
-          @job.perform
-          exit! 0
-        rescue Exception => e
-          logger.warn(process_name) { e }
-          exit! 1
-        end
-      end
+      }
+    end
+
+    def stop
+      @stopped = true
+    end
+
+    def atfork_parent
+      output.close
+    end
+
+    def atfork_child
+      to_master.close
+    end
+
+    private
+
+    def set_up?
+      @set_up
+    end
+
+    def setup_done
+      @set_up = true
     end
 
     def trap_signals
@@ -53,20 +78,9 @@ module Sisyphus
       end
     end
 
-    def stop
-      @stopped = true
-    end
-
     def stopped?
       @stopped
     end
 
-    def process_name=(name)
-      $0 = name
-    end
-
-    def process_name
-      $0
-    end
   end
 end

data/lib/sisyphus/worker_pool.rb

@@ -0,0 +1,36 @@
+require_relative './worker'
+
+module Sisyphus
+  class WorkerPool
+
+    attr_reader :workers, :worker_factory
+
+    def initialize(worker_factory)
+      @worker_factory = worker_factory
+      @workers = []
+    end
+
+    def spawn_worker
+      worker = create_worker
+      if wpid = fork
+        worker.atfork_parent
+        workers << { pid: wpid, reader: worker.to_master }
+      else
+        worker.atfork_child
+        self.process_name = "Worker #{Process.pid}"
+        worker.start
+      end
+    end
+
+    private
+
+    def create_worker
+      worker_factory.create_worker
+    end
+
+    def process_name=(name)
+      $0 = name
+    end
+
+  end
+end

data/lib/sisyphus.rb

@@ -1,6 +1,6 @@
 require "version"
+require 'sisyphus/simple_execution_strategy'
 require 'sisyphus/job'
-require 'sisyphus/worker'
 require 'sisyphus/master'
 
 module Sisyphus

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Sisyphus
-  VERSION = "0.2.3"
+  VERSION = "0.2.5"
 end

data/sisyphus.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 1.9.3'
+
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 2.14"

data/spec/sisyphus/forking_execution_strategy_spec.rb

@@ -0,0 +1,98 @@
+require 'spec_helper'
+require_relative '../../lib/sisyphus/forking_execution_strategy'
+
+module Sisyphus
+  describe ForkingExecutionStrategy do
+
+    let(:job) { double :job }
+    let(:error_handler) { ->(name, raised_error) {} }
+    let(:strategy) { ForkingExecutionStrategy.new }
+    let(:child_pid) { 1 }
+    let(:status) { double :process_status }
+
+    it 'forks on execution' do
+      allow(Process).to receive(:waitpid2).with(child_pid) { [child_pid, status] }
+      allow(status).to receive(:success?) { true }
+      expect(strategy).to receive(:fork) { child_pid }
+      strategy.execute job
+    end
+
+    context 'in the parent process' do
+
+      before :each do
+        allow(strategy).to receive(:fork) { child_pid }
+        allow(status).to receive(:success?) { true }
+      end
+
+      it 'waits for the execution to finish' do
+        expect(Process).to receive(:waitpid2).with(child_pid) { [child_pid, status] }
+        strategy.execute(job, error_handler)
+      end
+
+      it 'gets the status of the child process' do
+        allow(Process).to receive(:waitpid2) { [child_pid, status] }
+        expect(strategy.execute job, error_handler).to eq(true)
+      end
+
+    end
+
+    context 'in the execution process' do
+
+      let(:ppid) { 2 }
+
+      before :each do
+        allow(strategy).to receive(:fork) { nil }
+        allow(job).to receive(:perform)
+        allow(strategy).to receive(:exit!)
+      end
+
+      it 'updates the process name' do
+        allow(Process).to receive(:ppid) { ppid }
+        expect(strategy).to receive(:process_name=).with("Child of worker #{ppid}")
+        strategy.execute job
+      end
+
+      it 'performs the job' do
+        expect(job).to receive(:perform)
+        strategy.execute job
+      end
+
+      it 'exits with a 0 status if job is performed without failing' do
+        expect(strategy).to receive(:exit!).with(0)
+        strategy.execute job
+      end
+
+      it 'does not call error_handler if execution is successful' do
+        allow(strategy).to receive(:exit!).with(0)
+        strategy.execute job, ->(n, e) { fail "Should not be called" }
+      end
+
+      context 'when the job#perform fails' do
+
+        let(:process_name) { "foobarbaz" }
+        let(:exception) { Exception.new("foo") }
+
+        before :each do
+          allow(strategy).to receive(:process_name) { process_name }
+          allow(job).to receive(:perform).and_raise(exception)
+        end
+
+        it 'exits with a 1 status if job is performed and it fails' do
+          expect(strategy).to receive(:exit!).with(1)
+          strategy.execute job, error_handler
+        end
+
+        it 'calls error_handler if execution is unsuccessful' do
+          allow(strategy).to receive(:exit!)
+          strategy.execute job, ->(name, raised_error) {
+            expect(name).to eq(process_name)
+            expect(raised_error).to eq(exception)
+          }
+        end
+
+      end
+
+    end
+
+  end
+end