gorgon 0.11.0 → 0.11.1

data/lib/gorgon/originator_protocol.rb

@@ -4,113 +4,115 @@ require 'gorgon/job_definition'
 require 'amqp'
 require 'uuidtools'
 
-class OriginatorProtocol
-  def initialize(logger, cluster_id=nil)
-    @originator_exchange_name = OriginatorProtocol.originator_exchange_name(cluster_id)
-    @job_exchange_name = OriginatorProtocol.job_exchange_name(cluster_id)
-    @logger = logger
-  end
+module Gorgon
+  class OriginatorProtocol
+    def initialize(logger, cluster_id=nil)
+      @originator_exchange_name = OriginatorProtocol.originator_exchange_name(cluster_id)
+      @job_exchange_name = OriginatorProtocol.job_exchange_name(cluster_id)
+      @logger = logger
+    end
 
-  def self.originator_exchange_name(cluster_id)
-    if cluster_id
-      "gorgon.originators.#{cluster_id}"
-    else
-      "gorgon.originators"
+    def self.originator_exchange_name(cluster_id)
+      if cluster_id
+        "gorgon.originators.#{cluster_id}"
+      else
+        "gorgon.originators"
+      end
     end
-  end
 
-  def self.job_exchange_name(cluster_id)
-    if cluster_id
-      "gorgon.jobs.#{cluster_id}"
-    else
-      'gorgon.jobs'
+    def self.job_exchange_name(cluster_id)
+      if cluster_id
+        "gorgon.jobs.#{cluster_id}"
+      else
+        'gorgon.jobs'
+      end
     end
-  end
 
-  def connect connection_information, options={}
-    @connection = AMQP.connect(connection_information)
-    @channel = AMQP::Channel.new(@connection)
-    @connection.on_closed { options[:on_closed].call } if options[:on_closed]
-    open_queues
-  end
+    def connect connection_information, options={}
+      @connection = AMQP.connect(connection_information)
+      @channel = AMQP::Channel.new(@connection)
+      @connection.on_closed { options[:on_closed].call } if options[:on_closed]
+      open_queues
+    end
 
-  def publish_files files
-    @file_queue = @channel.queue("file_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+    def publish_files files
+      @file_queue = @channel.queue("file_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
 
-    files.each do |file|
-      @channel.default_exchange.publish(file, :routing_key => @file_queue.name)
+      files.each do |file|
+        @channel.default_exchange.publish(file, :routing_key => @file_queue.name)
+      end
     end
-  end
 
-  def publish_job_to_all job_definition
-    job_definition = append_protocol_information_to_job_definition(job_definition)
-    @channel.fanout(@job_exchange_name).publish(job_definition.to_json)
-  end
+    def publish_job_to_all job_definition
+      job_definition = append_protocol_information_to_job_definition(job_definition)
+      @channel.fanout(@job_exchange_name).publish(job_definition.to_json)
+    end
 
-  def publish_job_to_one job_definition, listener_queue_name
-    job_definition = append_protocol_information_to_job_definition(job_definition)
-    @channel.default_exchange.publish(job_definition.to_json, :routing_key => listener_queue_name)
-  end
+    def publish_job_to_one job_definition, listener_queue_name
+      job_definition = append_protocol_information_to_job_definition(job_definition)
+      @channel.default_exchange.publish(job_definition.to_json, :routing_key => listener_queue_name)
+    end
 
-  def append_protocol_information_to_job_definition job_definition
-    job_definition = job_definition.dup
+    def append_protocol_information_to_job_definition job_definition
+      job_definition = job_definition.dup
 
-    job_definition.file_queue_name = @file_queue.name
-    job_definition.reply_exchange_name = @reply_exchange.name
+      job_definition.file_queue_name = @file_queue.name
+      job_definition.reply_exchange_name = @reply_exchange.name
 
-    return job_definition
-  end
+      return job_definition
+    end
 
-  def send_message_to_listeners type, body={}
-    # TODO: we probably want to use a different exchange for this type of messages
-    message = {:type => type, :reply_exchange_name => @reply_exchange.name, :body => body}
-    @channel.fanout(@job_exchange_name).publish(Yajl::Encoder.encode(message))
-  end
+    def send_message_to_listeners type, body={}
+      # TODO: we probably want to use a different exchange for this type of messages
+      message = {:type => type, :reply_exchange_name => @reply_exchange.name, :body => body}
+      @channel.fanout(@job_exchange_name).publish(Yajl::Encoder.encode(message))
+    end
 
-  def receive_payloads
-    @reply_queue.subscribe do |payload|
-      yield payload
+    def receive_payloads
+      @reply_queue.subscribe do |payload|
+        yield payload
+      end
     end
-  end
 
-  def receive_new_listener_notifications
-    @originator_queue.subscribe do |payload|
-      yield payload
+    def receive_new_listener_notifications
+      @originator_queue.subscribe do |payload|
+        yield payload
+      end
     end
-  end
 
-  def cancel_job
-    @file_queue.purge if @file_queue
-    @channel.fanout("gorgon.worker_managers").publish(cancel_message) if @channel
-    @logger.log "Cancel Message sent"
-  end
+    def cancel_job
+      @file_queue.purge if @file_queue
+      @channel.fanout("gorgon.worker_managers").publish(cancel_message) if @channel
+      @logger.log "Cancel Message sent"
+    end
 
-  def disconnect
-    cleanup_queues_and_exchange
-    @connection.disconnect if @connection
-  end
+    def disconnect
+      cleanup_queues_and_exchange
+      @connection.disconnect if @connection
+    end
 
-  private
+    private
 
-  def open_queues
-    @reply_queue = @channel.queue("reply_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
-    @reply_exchange = @channel.direct("reply_exchange_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
-    @reply_queue.bind(@reply_exchange)
+    def open_queues
+      @reply_queue = @channel.queue("reply_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+      @reply_exchange = @channel.direct("reply_exchange_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+      @reply_queue.bind(@reply_exchange)
 
-    # Provides a way for new listeners to announce their presence to originators that have already started the job
-    @originator_queue = @channel.queue("originator_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
-    @originator_exchange = @channel.fanout(@originator_exchange_name)
-    @originator_queue.bind(@originator_exchange)
-  end
+      # Provides a way for new listeners to announce their presence to originators that have already started the job
+      @originator_queue = @channel.queue("originator_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+      @originator_exchange = @channel.fanout(@originator_exchange_name)
+      @originator_queue.bind(@originator_exchange)
+    end
 
-  def cleanup_queues_and_exchange
-    @reply_queue.delete if @reply_queue
-    @file_queue.delete if @file_queue
-    @reply_exchange.delete if @reply_exchange
-    @originator_queue.delete if @originator_queue
-  end
+    def cleanup_queues_and_exchange
+      @reply_queue.delete if @reply_queue
+      @file_queue.delete if @file_queue
+      @reply_exchange.delete if @reply_exchange
+      @originator_queue.delete if @originator_queue
+    end
 
-  def cancel_message
-    Yajl::Encoder.encode({:action => "cancel_job"})
+    def cancel_message
+      Yajl::Encoder.encode({:action => "cancel_job"})
+    end
   end
 end

data/lib/gorgon/ping_service.rb

@@ -5,63 +5,65 @@ require 'gorgon/colors'
 
 require 'colorize'
 
-class PingService
-  include Configuration
+module Gorgon
+  class PingService
+    include Configuration
 
-  TIMEOUT=4
+    TIMEOUT=4
 
-  def initialize
-    @configuration = load_configuration_from_file("gorgon.json")
-    @logger = OriginatorLogger.new @configuration[:originator_log_file]
-    @protocol = OriginatorProtocol.new @logger
-    @listeners = []
-  end
+    def initialize
+      @configuration = load_configuration_from_file("gorgon.json", merge: "gorgon_secret.json")
+      @logger = OriginatorLogger.new @configuration[:originator_log_file]
+      @protocol = OriginatorProtocol.new @logger
+      @listeners = []
+    end
 
-  def ping_listeners
-    Signal.trap("INT") { disconnect }
-    Signal.trap("TERM") { disconnect }
+    def ping_listeners
+      Signal.trap("INT") { disconnect }
+      Signal.trap("TERM") { disconnect }
 
-    EventMachine.run do
-      @logger.log "Connecting..."
-      @protocol.connect @configuration[:connection],  :on_closed => proc {EM.stop}
+      EventMachine.run do
+        @logger.log "Connecting..."
+        @protocol.connect @configuration[:connection],  :on_closed => proc {EM.stop}
 
-      @logger.log "Pinging Listeners..."
-      @protocol.send_message_to_listeners :ping
+        @logger.log "Pinging Listeners..."
+        @protocol.send_message_to_listeners :ping
 
-      EM.add_timer(TIMEOUT) { disconnect }
+        EM.add_timer(TIMEOUT) { disconnect }
 
-      @protocol.receive_payloads do |payload|
-        @logger.log "Received #{payload}"
+        @protocol.receive_payloads do |payload|
+          @logger.log "Received #{payload}"
 
-        handle_reply(Yajl::Parser.new(:symbolize_keys => true).parse(payload))
+          handle_reply(Yajl::Parser.new(:symbolize_keys => true).parse(payload))
+        end
       end
     end
-  end
 
-  private
+    private
 
-  def disconnect
-    @protocol.disconnect
-    print_summary
-  end
-
-  def handle_reply payload
-    if payload[:type] != "ping_response"
-      puts "Unexpected message received: #{payload}"
-      return
+    def disconnect
+      @protocol.disconnect
+      print_summary
     end
 
-    @listeners << payload
-    hostname = payload[:hostname].colorize(Colors::HOST)
-    puts "#{hostname} is running Listener version #{payload[:version]} and uses #{payload[:worker_slots]} workers"
-  end
+    def handle_reply payload
+      if payload[:type] != "ping_response"
+        puts "Unexpected message received: #{payload}"
+        return
+      end
 
-  def print_summary
-    worker_slots = @listeners.inject(0) { |sum, l| sum + l[:worker_slots] }
-    puts "\n#{@listeners.size} host(s) responded using a total of #{worker_slots} worker(s)."
-  end
+      @listeners << payload
+      hostname = payload[:hostname].colorize(Colors::HOST)
+      puts "#{hostname} is running Listener version #{payload[:version]} and uses #{payload[:worker_slots]} workers"
+    end
 
-  def on_disconnect
-    EventMachine.stop
+    def print_summary
+      worker_slots = @listeners.inject(0) { |sum, l| sum + l[:worker_slots] }
+      puts "\n#{@listeners.size} host(s) responded using a total of #{worker_slots} worker(s)."
+    end
+
+    def on_disconnect
+      EventMachine.stop
+    end
   end
 end

data/lib/gorgon/pipe_forker.rb

@@ -1,22 +1,24 @@
-module PipeForker
-  def pipe_fork
-    stdin = Pipe.new(*IO.pipe)
-    pid = fork do
-      stdin.write.close
-      STDIN.reopen(stdin.read)
-      stdin.read.close
+module Gorgon
+  module PipeForker
+    def pipe_fork
+      stdin = Pipe.new(*IO.pipe)
+      pid = fork do
+        stdin.write.close
+        STDIN.reopen(stdin.read)
+        stdin.read.close
 
-      yield
+        yield
 
-      exit
-    end
+        exit
+      end
 
-    stdin.read.close
+      stdin.read.close
 
-    return pid, stdin.write
-  end
+      return pid, stdin.write
+    end
 
-  private
+    private
 
-  Pipe = Struct.new(:read, :write)
+    Pipe = Struct.new(:read, :write)
+  end
 end

data/lib/gorgon/progress_bar_view.rb

@@ -2,142 +2,144 @@ require 'gorgon/colors'
 
 require 'ruby-progressbar'
 require 'colorize'
+module Gorgon
+  class ProgressBarView
+
+    MAX_LENGTH = 200
+    LOADING_MSG = "Loading environment and workers..."
+    RUNNING_MSG = "Running files:"
+    LEGEND_MSG = "Legend:\nF - failure files count\nH - number of hosts that have run files\nW - number of workers running files"
+
+    PROGRESS_BAR_REFRESH_RATE = 0.5
+    # - because the resolution of the elapsed time is one second, we use the nyquist frequency of 0.5 seconds
+    # http://en.wikipedia.org/wiki/Nyquist_frequency
+
+    def initialize job_state
+      @job_state = job_state
+      @job_state.add_observer(self)
+      @timer = EventMachine::PeriodicTimer.new(PROGRESS_BAR_REFRESH_RATE) { update_elapsed_time }
+    end
 
-MAX_LENGTH = 200
-LOADING_MSG = "Loading environment and workers..."
-RUNNING_MSG = "Running files:"
-LEGEND_MSG = "Legend:\nF - failure files count\nH - number of hosts that have run files\nW - number of workers running files"
+    def show
+      print LOADING_MSG
+    end
 
-PROGRESS_BAR_REFRESH_RATE = 0.5
-# - because the resolution of the elapsed time is one second, we use the nyquist frequency of 0.5 seconds
-# http://en.wikipedia.org/wiki/Nyquist_frequency
+    def update payload={}
+      output_gorgon_crash_message payload if gorgon_crashed? payload
 
-class ProgressBarView
-  def initialize job_state
-    @job_state = job_state
-    @job_state.add_observer(self)
-    @timer = EventMachine::PeriodicTimer.new(PROGRESS_BAR_REFRESH_RATE) { update_elapsed_time }
-  end
-
-  def show
-    print LOADING_MSG
-  end
+      create_progress_bar_if_started_job_running
 
-  def update payload={}
-    output_gorgon_crash_message payload if gorgon_crashed? payload
+      return if @progress_bar.nil? || @finished
 
-    create_progress_bar_if_started_job_running
+      failed_files_count = @job_state.failed_files_count
 
-    return if @progress_bar.nil? || @finished
+      @progress_bar.title=" F: #{failed_files_count} H: #{@job_state.total_running_hosts} W: #{@job_state.total_running_workers}"
+      if failed_files_count > 0
+        @progress_bar.format(format(bar: :red, title: :default))
+      end
 
-    failed_files_count = @job_state.failed_files_count
+      @progress_bar.progress = @job_state.finished_files_count
 
-    @progress_bar.title=" F: #{failed_files_count} H: #{@job_state.total_running_hosts} W: #{@job_state.total_running_workers}"
-    if failed_files_count > 0
-      @progress_bar.format(format(bar: :red, title: :default))
+      if @job_state.is_job_complete? || @job_state.is_job_cancelled?
+        @finished = true
+        print_summary
+      end
     end
 
-    @progress_bar.progress = @job_state.finished_files_count
-
-    if @job_state.is_job_complete? || @job_state.is_job_cancelled?
-      @finished = true
-      print_summary
+    def create_progress_bar_if_started_job_running
+      if @progress_bar.nil? && @job_state.state == :running
+        print "\r#{' ' * (LOADING_MSG.length)}\r"
+        puts LEGEND_MSG
+        @progress_bar = ProgressBar.create(:total => @job_state.total_files,
+                                           :length => [terminal_length, MAX_LENGTH].min,
+                                           :format => format(bar: :green, title: :white));
+      end
     end
-  end
 
-  def create_progress_bar_if_started_job_running
-    if @progress_bar.nil? && @job_state.state == :running
-      print "\r#{' ' * (LOADING_MSG.length)}\r"
-      puts LEGEND_MSG
-      @progress_bar = ProgressBar.create(:total => @job_state.total_files,
-                                         :length => [terminal_length, MAX_LENGTH].min,
-                                         :format => format(bar: :green, title: :white));
+    def update_elapsed_time
+      @timer.cancel if @finished
+      @progress_bar.refresh if @progress_bar
     end
-  end
-
-  def update_elapsed_time
-    @timer.cancel if @finished
-    @progress_bar.refresh if @progress_bar
-  end
 
-private
-  def gorgon_crashed? payload
-     payload[:type] == "crash" && payload[:action] != "finish"
-  end
-
-  def output_gorgon_crash_message payload
-    $stderr.puts "\nA #{'crash'.red} occurred at '#{payload[:hostname].colorize Colors::HOST}':"
-    $stderr.puts payload[:stdout].yellow unless payload[:stdout].to_s.strip.length == 0
-    $stderr.puts payload[:stderr].yellow unless payload[:stderr].to_s.strip.length == 0
-    if @progress_bar.nil?
-      print LOADING_MSG         # if still loading, print msg so user won't think the whole job crashed
+    private
+    def gorgon_crashed? payload
+      payload[:type] == "crash" && payload[:action] != "finish"
     end
-  end
 
-  def format colors
-    # TODO: decide what bar to use
-    #    bar = "%b>%i".colorize(colors[:bar])
-    bar = "%w>%i".colorize(colors[:bar])
-    title = "%t".colorize(colors[:title])
+    def output_gorgon_crash_message payload
+      $stderr.puts "\nA #{'crash'.red} occurred at '#{payload[:hostname].colorize Colors::HOST}':"
+      $stderr.puts payload[:stdout].yellow unless payload[:stdout].to_s.strip.length == 0
+      $stderr.puts payload[:stderr].yellow unless payload[:stderr].to_s.strip.length == 0
+      if @progress_bar.nil?
+        print LOADING_MSG         # if still loading, print msg so user won't think the whole job crashed
+      end
+    end
 
-    "#{title} | [#{bar}] %c/%C %a"
-  end
+    def format colors
+      # TODO: decide what bar to use
+      #    bar = "%b>%i".colorize(colors[:bar])
+      bar = "%w>%i".colorize(colors[:bar])
+      title = "%t".colorize(colors[:title])
 
-  def terminal_length
-    `stty size`.split.map { |x| x.to_i }.reverse[0].to_i
-  end
+      "#{title} | [#{bar}] %c/%C %a"
+    end
 
-  def print_summary
-    print_failed_tests
-    print_running_files
-    #TODO: print other stats: time, total file, total failures, etc
-  end
+    def terminal_length
+      `stty size`.split.map { |x| x.to_i }.reverse[0].to_i
+    end
 
-  def print_failed_tests
-    @job_state.each_failed_test do |test|
-      puts "\n" + ('*' * 80).magenta #light_red
-      puts("File '#{test[:filename].colorize(Colors::FILENAME)}' failed/crashed at " \
-           + "'#{test[:hostname].colorize(Colors::HOST)}:#{test[:worker_id]}'\n")
-      msg = build_fail_message test[:failures]
-      puts "#{msg}\n"
+    def print_summary
+      print_failed_tests
+      print_running_files
+      #TODO: print other stats: time, total file, total failures, etc
     end
-  end
 
-  def build_fail_message failures
-    result = []
-    failures.each do |failure|
-      if failure.is_a?(Hash)
-        result << build_fail_message_from_hash(failure)
-      else
-        result << build_fail_message_from_string(failure)
+    def print_failed_tests
+      @job_state.each_failed_test do |test|
+        puts "\n" + ('*' * 80).magenta #light_red
+        puts("File '#{test[:filename].colorize(Colors::FILENAME)}' failed/crashed at " \
+             + "'#{test[:hostname].colorize(Colors::HOST)}:#{test[:worker_id]}'\n")
+        msg = build_fail_message test[:failures]
+        puts "#{msg}\n"
       end
     end
 
-    result.join("\n")
-  end
+    def build_fail_message failures
+      result = []
+      failures.each do |failure|
+        if failure.is_a?(Hash)
+          result << build_fail_message_from_hash(failure)
+        else
+          result << build_fail_message_from_string(failure)
+        end
+      end
+
+      result.join("\n")
+    end
 
-  def print_running_files
-    title = "Unfinished files".yellow
-    puts "\n#{title} - The following files were still running:" if @job_state.total_running_workers > 0
+    def print_running_files
+      title = "Unfinished files".yellow
+      puts "\n#{title} - The following files were still running:" if @job_state.total_running_workers > 0
 
-    @job_state.each_running_file do |hostname, filename|
-      filename_str = filename.dup.colorize(Colors::FILENAME)
-      hostname_str = hostname.dup.colorize(Colors::HOST)
-      puts "\t#{filename_str} at '#{hostname_str}'"
+      @job_state.each_running_file do |hostname, filename|
+        filename_str = filename.dup.colorize(Colors::FILENAME)
+        hostname_str = hostname.dup.colorize(Colors::HOST)
+        puts "\t#{filename_str} at '#{hostname_str}'"
+      end
     end
-  end
 
-  def build_fail_message_from_string failure
-    result = failure.gsub(/^Error:/, "Error:".yellow)
-    result.gsub!(/^Failure:/, "Failure:".red)
-    result
-  end
+    def build_fail_message_from_string failure
+      result = failure.gsub(/^Error:/, "Error:".yellow)
+      result.gsub!(/^Failure:/, "Failure:".red)
+      result
+    end
 
-  def build_fail_message_from_hash failure
-    result = "#{'Test name'.light_yellow}: #{failure[:test_name]}"
-    result << "\n#{failure[:class].red}" if failure[:class]
-    result << "\n#{'Message:'.light_yellow} \n\t#{failure[:message]}" if failure[:message]
-    result << "\n"
-    result
+    def build_fail_message_from_hash failure
+      result = "#{'Test name'.light_yellow}: #{failure[:test_name]}"
+      result << "\n#{failure[:class].red}" if failure[:class]
+      result << "\n#{'Message:'.light_yellow} \n\t#{failure[:message]}" if failure[:message]
+      result << "\n"
+      result
+    end
   end
 end