gorgon 0.2.0 → 0.3.0

data/lib/gorgon/originator_protocol.rb

@@ -16,7 +16,7 @@ class OriginatorProtocol
   end
 
   def publish_files files
-    @file_queue = @channel.queue(UUIDTools::UUID.timestamp_create.to_s)
+    @file_queue = @channel.queue("file_queue_" + UUIDTools::UUID.timestamp_create.to_s)
 
     files.each do |file|
       @channel.default_exchange.publish(file, :routing_key => @file_queue.name)
@@ -30,9 +30,9 @@ class OriginatorProtocol
     @channel.fanout("gorgon.jobs").publish(job_definition.to_json)
   end
 
-  def ping_listeners
-    # TODO: we probably want to use a different exchange for pinging when we add more services
-    message = {:type => "ping", :reply_exchange_name => @reply_exchange.name}
+  def send_message_to_listeners type, body={}
+    # TODO: we probably want to use a different exchange for this type of messages
+    message = {:type => type, :reply_exchange_name => @reply_exchange.name, :body => body}
     @channel.fanout("gorgon.jobs").publish(Yajl::Encoder.encode(message))
   end
 
@@ -49,21 +49,22 @@ class OriginatorProtocol
   end
 
   def disconnect
-    cleanup_queues
+    cleanup_queues_and_exchange
     @connection.disconnect
   end
 
   private
 
   def open_queues
-    @reply_queue = @channel.queue(UUIDTools::UUID.timestamp_create.to_s)
-    @reply_exchange = @channel.direct(UUIDTools::UUID.timestamp_create.to_s)
+    @reply_queue = @channel.queue("reply_queue_" + UUIDTools::UUID.timestamp_create.to_s)
+    @reply_exchange = @channel.direct("reply_exchange_" + UUIDTools::UUID.timestamp_create.to_s)
     @reply_queue.bind(@reply_exchange)
   end
 
-  def cleanup_queues
+  def cleanup_queues_and_exchange
     @reply_queue.delete if @reply_queue
     @file_queue.delete if @file_queue
+    @reply_exchange.delete if @reply_exchange
   end
 
   def cancel_message

data/lib/gorgon/ping_service.rb

@@ -26,7 +26,7 @@ class PingService
       @protocol.connect @configuration[:connection],  :on_closed => proc {EM.stop}
 
       @logger.log "Pinging Listeners..."
-      @protocol.ping_listeners
+      @protocol.send_message_to_listeners :ping
 
       EM.add_timer(TIMEOUT) { disconnect }
 
@@ -53,7 +53,7 @@ class PingService
 
     @listeners << payload
     hostname = payload[:hostname].colorize(Colors::HOST)
-    puts "#{hostname} is running Listener version #{payload[:version]}"
+    puts "#{hostname} is running Listener version #{payload[:version]} and uses #{payload[:worker_slots]} workers"
   end
 
   def print_summary

data/lib/gorgon/pipe_forker.rb

@@ -0,0 +1,22 @@
+module PipeForker
+  def pipe_fork
+    stdin = Pipe.new(*IO.pipe)
+    pid = fork do
+      stdin.write.close
+      STDIN.reopen(stdin.read)
+      stdin.read.close
+
+      yield
+
+      exit
+    end
+
+    stdin.read.close
+
+    return pid, stdin.write
+  end
+
+  private
+
+  Pipe = Struct.new(:read, :write)
+end

data/lib/gorgon/source_tree_syncer.rb

@@ -25,9 +25,10 @@ class SourceTreeSyncer
     pid, stdin, stdout, stderr = Open4::popen4 @sys_command
     stdin.close
 
+    ignore, status = Process.waitpid2 pid
+
     @output, @errors = [stdout, stderr].map { |p| begin p.read ensure p.close end }
 
-    ignore, status = Process.waitpid2 pid
     @exitstatus = status.exitstatus
   end
 

data/lib/gorgon/version.rb

@@ -1,3 +1,3 @@
 module Gorgon
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/gorgon/worker.rb

@@ -2,6 +2,8 @@ require "gorgon/configuration"
 require "gorgon/amqp_service"
 require 'gorgon/callback_handler'
 require "gorgon/g_logger"
+require 'gorgon/job_definition'
+require "gorgon/testunit_runner"
 
 require "uuidtools"
 require "awesome_print"
@@ -9,7 +11,6 @@ require "socket"
 
 module WorkUnit
   def self.run_file filename
-    require "gorgon/testunit_runner"
     start_t = Time.now
 
     begin
@@ -31,29 +32,44 @@ end
 class Worker
   include GLogger
 
-  def self.build(config)
-    payload = Yajl::Parser.new(:symbolize_keys => true).parse($stdin.read)
-    job_definition = JobDefinition.new(payload)
+  class << self
+    def build(worker_id, config)
+      redirect_output_to_files worker_id
 
-    connection_config = config[:connection]
-    amqp = AmqpService.new connection_config
+      payload = Yajl::Parser.new(:symbolize_keys => true).parse($stdin.read)
+      job_definition = JobDefinition.new(payload)
 
-    callback_handler = CallbackHandler.new(job_definition.callbacks)
+      connection_config = config[:connection]
+      amqp = AmqpService.new connection_config
 
-    worker_id = UUIDTools::UUID.timestamp_create.to_s
-    ENV["GORGON_WORKER_ID"] = worker_id
+      callback_handler = CallbackHandler.new(job_definition.callbacks)
 
-    params = {
-      :amqp => amqp,
-      :file_queue_name => job_definition.file_queue_name,
-      :reply_exchange_name => job_definition.reply_exchange_name,
-      :worker_id => worker_id,
-      :test_runner => WorkUnit,
-      :callback_handler => callback_handler,
-      :log_file => config[:log_file]
-    }
+      ENV["GORGON_WORKER_ID"] = worker_id.to_s
 
-    new(params)
+      params = {
+        :amqp => amqp,
+        :file_queue_name => job_definition.file_queue_name,
+        :reply_exchange_name => job_definition.reply_exchange_name,
+        :worker_id => worker_id,
+        :test_runner => WorkUnit,
+        :callback_handler => callback_handler,
+        :log_file => config[:log_file]
+      }
+
+      new(params)
+    end
+
+    def output_file id, stream
+      "/tmp/gorgon-worker-#{id}.#{stream.to_s}"
+    end
+
+    def redirect_output_to_files worker_id
+      STDOUT.reopen(File.open(output_file(worker_id, :out), 'w'))
+      STDOUT.sync = true
+
+      STDERR.reopen(File.open(output_file(worker_id, :err), 'w'))
+      STDERR.sync = true
+    end
   end
 
   def initialize(params)
@@ -68,27 +84,35 @@ class Worker
   end
 
   def work
-    log "Running before_start callback..."
-    @callback_handler.before_start
-
-    log "Running files ..."
-    @amqp.start_worker @file_queue_name, @reply_exchange_name do |queue, exchange|
-      while filename = queue.pop
-        exchange.publish make_start_message(filename)
-        log "Running '#{filename}'"
-        test_results = run_file(filename)
-        exchange.publish make_finish_message(filename, test_results)
+    begin
+      log "Running before_start callback..."
+      register_trap_ints        # do it before calling before_start callback!
+      @callback_handler.before_start
+      @cleaned = false
+
+      log "Running files ..."
+      @amqp.start_worker @file_queue_name, @reply_exchange_name do |queue, exchange|
+        while filename = queue.pop
+          exchange.publish make_start_message(filename)
+          log "Running '#{filename}'"
+          test_results = run_file(filename)
+          exchange.publish make_finish_message(filename, test_results)
+        end
       end
+    rescue Exception => e
+      clean_up
+      raise e                     # So worker manager can catch it
     end
-  ensure # this 'ensure' that we run after_complete even after an 'INT' signal
     clean_up
   end
 
   private
 
   def clean_up
+    return if @cleaned
     log "Running after_complete callback"
     @callback_handler.after_complete
+    @cleaned = true
   end
 
   def run_file(filename)
@@ -102,4 +126,14 @@ class Worker
   def make_finish_message(filename, results)
     {:action => :finish, :hostname => Socket.gethostname, :worker_id => @worker_id, :filename => filename}.merge(results)
   end
+
+  def register_trap_ints
+    Signal.trap("INT") { ctrl_c }
+    Signal.trap("TERM") { ctrl_c }
+  end
+
+  def ctrl_c
+    clean_up
+    exit
+  end
 end

data/lib/gorgon/worker_manager.rb

@@ -1,22 +1,37 @@
 require "gorgon/worker"
 require "gorgon/g_logger"
 require 'gorgon/callback_handler'
-require 'gorgon/pipe_manager'
+require 'gorgon/pipe_forker'
 require 'gorgon/job_definition'
+require "gorgon/crash_reporter"
 
 require 'eventmachine'
 
 class WorkerManager
-  include PipeManager
+  include PipeForker
   include GLogger
+  include CrashReporter
+
+  STDOUT_FILE='/tmp/gorgon-worker-mgr.out'
+  STDERR_FILE='/tmp/gorgon-worker-mgr.err'
 
   def self.build listener_config_file
     @listener_config_file = listener_config_file
     config = Configuration.load_configuration_from_file(listener_config_file)
 
+    redirect_output_to_files
+
     new config
   end
 
+  def self.redirect_output_to_files
+    STDOUT.reopen(File.open(STDOUT_FILE, 'w'))
+    STDOUT.sync = true
+
+    STDERR.reopen(File.open(STDERR_FILE, 'w'))
+    STDERR.sync = true
+  end
+
   def initialize config
     initialize_logger config[:log_file]
     log "Worker Manager #{Gorgon::VERSION} initializing"
@@ -68,7 +83,13 @@ class WorkerManager
     @available_worker_slots -= 1
     ENV["GORGON_CONFIG_PATH"] = @listener_config_filename
 
-    pid, stdin, stdout, stderr = pipe_fork_worker
+    worker_id = get_worker_id
+    log "Forking Worker #{worker_id}"
+    pid, stdin = pipe_fork do
+      worker = Worker.build(worker_id, @config)
+      worker.work
+    end
+
     @worker_pids << pid
     stdin.write(@job_definition.to_json)
     stdin.close
@@ -82,17 +103,18 @@ class WorkerManager
 
     worker_complete = proc do |status|
       if status.exitstatus != 0
-        log_error "Worker #{pid} crashed with exit status #{status.exitstatus}!"
-        error_msg = stderr.read
-        log_error "ERROR MSG: #{error_msg}"
+        exitstatus = status.exitstatus
+        log_error "Worker #{pid} crashed with exit status #{exitstatus}!"
 
         # originator may have cancel job and exit, so only try to send message
         begin
-          reply = {:type => :crash,
-            :hostname => Socket.gethostname,
-            :stdout => stdout.read,
-            :stderr => error_msg}
-          @reply_exchange.publish(Yajl::Encoder.encode(reply))
+          out_file = Worker.output_file(worker_id, :out)
+          err_file = Worker.output_file(worker_id, :err)
+
+          msg = report_crash @reply_exchange, :out_file => out_file,
+          :err_file => err_file, :footer_text => footer_text(err_file, out_file)
+          log_error "Process output:\n#{msg}"
+
           # TODO: find a way to stop the whole system when a worker crashes or do something more clever
         rescue Exception => e
           log_error "Exception raised when trying to report crash to originator:"
@@ -105,6 +127,10 @@ class WorkerManager
     EventMachine.defer(watcher, worker_complete)
   end
 
+  def get_worker_id
+    @worker_id_count = @worker_id_count.nil? ? 1 : @worker_id_count + 1
+  end
+
   def on_worker_complete
     @available_worker_slots += 1
     on_current_job_complete if current_job_complete?
@@ -125,7 +151,7 @@ class WorkerManager
     @bunny.stop
   end
 
-  CANCEL_TIMEOUT = 15
+  CANCEL_TIMEOUT = 20
   def subscribe_to_originator_queue
 
     originator_watcher = proc do
@@ -154,4 +180,8 @@ class WorkerManager
 
     EventMachine.defer(originator_watcher, handle_message)
   end
+
+  def footer_text err_file, out_file
+    "\n***** See #{err_file} and #{out_file} at '#{Socket.gethostname}' for more details *****\n"
+  end
 end

data/spec/crash_reporter_spec.rb

@@ -0,0 +1,46 @@
+require 'gorgon/crash_reporter'
+
+describe "CrashReporter" do
+  let(:exchange) { stub("Bunny Exchange", :publish => nil) }
+  let(:info) { {
+      :out_file => "stdout_file", :err_file => "stderr_file", :footer_text => "Text"
+    } }
+
+  let(:container_class) do
+    Class.new do
+      extend(CrashReporter)
+    end
+  end
+
+  describe "#report_crash" do
+    it "tails output files to get last few lines" do
+      container_class.should_receive(:'`').once.
+        with(/tail.*stdout_file/).and_return ""
+      container_class.should_receive(:'`').once.
+        with(/tail.*stderr_file/).and_return ""
+      container_class.report_crash exchange, info
+    end
+
+    it "calls send_crash_message" do
+      container_class.stub!(:'`').and_return "stdout text", "stderr text "
+      container_class.should_receive(:send_crash_message).with(exchange, "stdout text", "stderr text Text")
+      container_class.report_crash exchange, info
+    end
+
+    it "returns last lines of output from stderr message and footer text" do
+      container_class.stub!(:'`').and_return "stdout text", "stderr text "
+      container_class.stub!(:send_crash_message)
+      result = container_class.report_crash exchange, info
+      result.should == "stdout text\nstderr text Text"
+    end
+  end
+
+  describe "#send_crash_message" do
+    it "sends message with output and errors from syncer using reply_exchange " do
+      reply = {:type => :crash, :hostname => Socket.gethostname, :stdout => "some output",
+        :stderr => "some errors"}
+      exchange.should_receive(:publish).with(Yajl::Encoder.encode(reply))
+      container_class.send_crash_message exchange, "some output", "some errors"
+    end
+  end
+end

data/spec/gem_command_handler_spec.rb

@@ -0,0 +1,84 @@
+require 'gorgon/gem_command_handler'
+
+describe GemCommandHandler do
+  let(:exchange) { stub("Bunny Exchange", :publish => nil) }
+  let(:bunny) { stub("Bunny", :exchange => exchange, :stop => nil) }
+
+  let(:payload) {
+      {:type => :update, :reply_exchange_name => "name",
+                                         :body => {:gem_command => "cmd"}}
+  }
+  let(:stdin) { stub("IO object", :close => nil)}
+  let(:stdout) { stub("IO object", :read => "output", :close => nil)}
+  let(:stderr) { stub("IO object", :read => "errors", :close => nil)}
+  let(:status) { stub("Process Status", :exitstatus => 0)}
+
+  describe "#handle" do
+    before do
+      @handler = GemCommandHandler.new bunny
+      @running_response = {:type => :running_command, :hostname => Socket.gethostname}
+      stub_methods
+    end
+
+    it "publishes 'running_command' message" do
+      bunny.should_receive(:exchange).with("name", anything).and_return(exchange)
+      Yajl::Encoder.should_receive(:encode).with(@running_response).and_return :json_msg
+      exchange.should_receive(:publish).with(:json_msg)
+      @handler.handle payload, {}
+    end
+
+    it "runs 'gem <command> gorgon' using popen4" do
+      Open4.should_receive(:popen4).with("gem cmd gorgon").and_return([1, stdin, stdout, stderr])
+      @handler.handle payload, {}
+    end
+
+    it "uses binary gem from configuration if this was specified" do
+      Open4.should_receive(:popen4).with("path/to/gem cmd gorgon").and_return([1, stdin, stdout, stderr])
+      @handler.handle payload, {:bin_gem_path => "path/to/gem"}
+    end
+
+    it "waits for the command to finish" do
+      Open4.should_receive(:popen4).ordered
+      Process.should_receive(:waitpid2).with(1).ordered.and_return([nil, status])
+      @handler.handle payload, {}
+    end
+
+    it "closes stding and reads stdout and stderr from process" do
+      stdin.should_receive(:close)
+      stdout.should_receive(:read)
+      stderr.should_receive(:read)
+      @handler.handle payload, {}
+    end
+
+    it "sends 'command_completed' message when exitstatus is 0" do
+      response = {:type => :command_completed, :hostname => Socket.gethostname,
+        :command => "gem cmd gorgon", :stdout => "output", :stderr => "errors" }
+      Yajl::Encoder.should_receive(:encode).once.ordered.with(@running_response)
+      Yajl::Encoder.should_receive(:encode).once.ordered.with(response).and_return :json_msg
+      exchange.should_receive(:publish).with(:json_msg)
+      @handler.handle payload, {}
+    end
+
+    it "stops bunny and exit if exitstatus is 0" do
+      bunny.should_receive(:stop).once.ordered
+      @handler.should_receive(:exit).once.ordered
+      @handler.handle payload, {}
+    end
+
+    it "sends 'command_failed' message when exitstatus is not 0" do
+      status.should_receive(:exitstatus).and_return(99)
+      response = {:type => :command_failed, :hostname => Socket.gethostname,
+        :command => "gem cmd gorgon", :stdout => "output", :stderr => "errors" }
+      Yajl::Encoder.should_receive(:encode).with(response).and_return :json_msg
+      exchange.should_receive(:publish).with(:json_msg)
+      @handler.handle payload, {}
+    end
+  end
+
+  def stub_methods
+    Open4.stub!(:popen4).and_return([1, stdin, stdout, stderr])
+    Process.stub!(:waitpid2).and_return([nil, status])
+    Yajl::Encoder.stub!(:encode).and_return :json_msg
+    @handler.stub!(:exit)
+  end
+end