gorgon 0.0.1

data/lib/gorgon/listener.rb

@@ -0,0 +1,147 @@
+require "gorgon/job_definition"
+require "gorgon/configuration"
+require 'gorgon/source_tree_syncer'
+require "gorgon/g_logger"
+require "gorgon/callback_handler"
+
+require "yajl"
+require "bunny"
+require "awesome_print"
+require "open4"
+require "tmpdir"
+require "socket"
+
+class Listener
+  include Configuration
+  include GLogger
+
+  def initialize
+    @listener_config_filename = Dir.pwd + "/gorgon_listener.json"
+    initialize_logger configuration[:log_file]
+
+    log "Listener initialized"
+    connect
+    initialize_personal_job_queue
+  end
+
+  def listen
+    log "Waiting for jobs..."
+    while true
+      sleep 2 unless poll
+    end
+  end
+
+  def connect
+    @bunny = Bunny.new(connection_information)
+    @bunny.start
+  end
+
+  def initialize_personal_job_queue
+    @job_queue = @bunny.queue("", :exclusive => true)
+    exchange = @bunny.exchange("gorgon.jobs", :type => :fanout)
+    @job_queue.bind(exchange)
+  end
+
+  def poll
+    message = @job_queue.pop
+    return false if message[:payload] == :queue_empty
+
+    run_job(message[:payload])
+
+    log "Waiting for more jobs..."
+    return true
+  end
+
+  def run_job(json_payload)
+    log "Job received: #{json_payload}"
+    payload = Yajl::Parser.new(:symbolize_keys => true).parse(json_payload)
+    @job_definition = JobDefinition.new(payload)
+    @reply_exchange = @bunny.exchange(@job_definition.reply_exchange_name)
+
+    @callback_handler = CallbackHandler.new(@job_definition.callbacks)
+    copy_source_tree(@job_definition.source_tree_path, @job_definition.sync_exclude)
+
+    if !run_after_sync
+      clean_up
+      return
+    end
+
+    Bundler.with_clean_env do
+      fork_worker_manager
+    end
+
+    clean_up
+  end
+
+  private
+
+  def run_after_sync
+    log "Running after_sync callback"
+    begin
+      @callback_handler.after_sync
+    rescue Exception => e
+      log_error "Exception raised when running after_sync callback_handler. Please, check your script in #{@job_definition.callbacks[:after_sync]}:"
+      log_error e.message
+      log_error "\n" + e.backtrace.join("\n")
+
+      reply = {:type => :crash,
+        :hostname => Socket.gethostname,
+        :message => "after_sync callback failed. Please, check your script in #{@job_definition.callbacks[:after_sync]}. Message: #{e.message}",
+        :backtrace => e.backtrace.join("\n")
+      }
+      @reply_exchange.publish(Yajl::Encoder.encode(reply))
+      return false
+    end
+    true
+  end
+
+  def copy_source_tree source_tree_path, exclude
+    log "Downloading source tree to temp directory..."
+    @syncer = SourceTreeSyncer.new source_tree_path
+    @syncer.exclude = exclude
+    if @syncer.sync
+      log "Command '#{@syncer.sys_command}' completed successfully."
+    else
+      #TODO handle error:
+      # - Discard job
+      # - Let the originator know about the error
+      # - Wait for the next job
+      log_error "Command '#{@syncer.sys_command}' failed!"
+    end
+  end
+
+  def clean_up
+    @syncer.remove_temp_dir
+  end
+
+  def fork_worker_manager
+    log "Forking Worker Manager"
+    ENV["GORGON_CONFIG_PATH"] = @listener_config_filename
+    pid, stdin, stdout, stderr = Open4::popen4 "bundle exec gorgon manage_workers"
+    stdin.write(@job_definition.to_json)
+    stdin.close
+
+    ignore, status = Process.waitpid2 pid
+    log "Worker Manager #{pid} finished"
+
+    if status.exitstatus != 0
+      log_error "Worker Manager #{pid} crashed with exit status #{status.exitstatus}!"
+      error_msg = stderr.read
+      log_error "ERROR MSG: #{error_msg}"
+
+      reply = {:type => :crash,
+        :hostname => Socket.gethostname,
+        :stdout => stdout.read,
+        :stderr => error_msg}
+      @reply_exchange.publish(Yajl::Encoder.encode(reply))
+    end
+  end
+
+  def connection_information
+    configuration[:connection]
+  end
+
+  def configuration
+    @configuration ||= load_configuration_from_file("gorgon_listener.json")
+  end
+end

data/lib/gorgon/originator.rb

@@ -0,0 +1,120 @@
+require 'gorgon/originator_protocol'
+require 'gorgon/configuration'
+require 'gorgon/job_state'
+require 'gorgon/progress_bar_view'
+require 'gorgon/originator_logger'
+require 'gorgon/failures_printer'
+
+require 'awesome_print'
+
+class Originator
+  include Configuration
+
+  def initialize
+    @configuration = nil
+  end
+
+  def originate
+    begin
+      Signal.trap("INT") { ctrl_c }
+      Signal.trap("TERM") { ctrl_c }
+
+      publish
+      @logger.log "Originator finished successfully"
+    rescue Exception
+      puts "Unhandled exception in originator:"
+      puts $!.message
+      puts $!.backtrace.join("\n")
+      puts "----------------------------------"
+      puts "Now attempting to cancel the job."
+      @logger.log_error "Unhandled Exception!"
+      cancel_job
+    end
+  end
+
+  def ctrl_c
+    puts "\nCtrl-C received! Just wait a moment while I clean up..."
+    cancel_job
+  end
+
+  def cancel_job
+    @protocol.cancel_job
+    @job_state.cancel
+
+    @protocol.disconnect
+  end
+
+  def publish
+    @logger = OriginatorLogger.new configuration[:originator_log_file]
+    @protocol = OriginatorProtocol.new @logger
+
+    EventMachine.run do
+      @logger.log "Connecting..."
+      @protocol.connect connection_information, :on_closed => method(:on_disconnect)
+
+      @logger.log "Publishing files..."
+      @protocol.publish_files files
+      create_job_state_and_observers
+
+      @logger.log "Publishing Job..."
+      @protocol.publish_job job_definition
+      @logger.log "Job Published"
+
+      @protocol.receive_payloads do |payload|
+        handle_reply(payload)
+      end
+    end
+  end
+
+  def cleanup_if_job_complete
+    if @job_state.is_job_complete?
+      @logger.log "Job is done"
+      @protocol.disconnect
+    end
+  end
+
+  def handle_reply(payload)
+    payload = Yajl::Parser.new(:symbolize_keys => true).parse(payload)
+
+    # at some point this will probably need to be fancy polymorphic type based responses, or at least a nice switch statement
+    if payload[:action] == "finish"
+      @job_state.file_finished payload
+    elsif payload[:action] == "start"
+      @job_state.file_started payload
+    end
+    @logger.log_message payload
+    # Uncomment this to see each message received by originator
+    # ap payload
+
+    cleanup_if_job_complete
+  end
+
+  def create_job_state_and_observers
+    @job_state = JobState.new files.count
+    @progress_bar_view = ProgressBarView.new @job_state
+    @progress_bar_view.show
+    failures_printer = FailuresPrinter.new @job_state
+  end
+
+  def on_disconnect
+    EventMachine.stop
+  end
+
+  def connection_information
+    configuration[:connection]
+  end
+
+  def files
+    @files ||= configuration[:files].reduce([]) do |memo, obj|
+      memo.concat(Dir[obj])
+    end.uniq
+  end
+
+  def job_definition
+    JobDefinition.new(@configuration[:job])
+  end
+
+  def configuration
+    @configuration ||= load_configuration_from_file("gorgon.json")
+  end
+end

data/lib/gorgon/originator_logger.rb

@@ -0,0 +1,36 @@
+require 'gorgon/g_logger'
+
+class OriginatorLogger
+  include GLogger
+
+  def initialize log_file
+    initialize_logger log_file
+  end
+
+  def log_message(payload)
+    if payload[:action] == "start"
+      log("Started running '#{payload[:filename]}' at '#{payload[:hostname]}'")
+    elsif payload[:action] == "finish"
+      print_finish(payload)
+    else # to be removed
+      ap payload
+    end
+  end
+
+  private
+
+  def print_finish(payload)
+    msg = "Finished running '#{payload[:filename]}' at '#{payload[:hostname]}'"
+    msg << failure_message(payload[:failures]) if payload[:type] == "fail"
+    log msg
+  end
+
+  def failure_message(failures)
+    msg = []
+    failures.each do |failure|
+      msg << failure
+    end
+    msg << ''
+    msg.join("\n")
+  end
+end

data/lib/gorgon/originator_protocol.rb

@@ -0,0 +1,65 @@
+require 'gorgon/job_definition'
+
+require 'amqp'
+require 'uuidtools'
+
+class OriginatorProtocol
+  def initialize logger
+    @logger = logger
+  end
+
+  def connect connection_information, options={}
+    @connection = AMQP.connect(connection_information)
+    @channel = AMQP::Channel.new(@connection)
+    @connection.on_closed { options[:on_closed].call } if options[:on_closed]
+    open_queues
+  end
+
+  def publish_files files
+    files.each do |file|
+      @channel.default_exchange.publish(file, :routing_key => @file_queue.name)
+    end
+  end
+
+  def publish_job job_definition
+    job_definition.file_queue_name = @file_queue.name
+    job_definition.reply_exchange_name = @reply_exchange.name
+
+    @channel.fanout("gorgon.jobs").publish(job_definition.to_json)
+  end
+
+  def receive_payloads
+    @reply_queue.subscribe do |payload|
+      yield payload
+    end
+  end
+
+  def cancel_job
+    @file_queue.purge
+    @channel.fanout("gorgon.worker_managers").publish(cancel_message)
+    @logger.log "Cancel Message sent"
+  end
+
+  def disconnect
+    cleanup_queues
+    @connection.disconnect
+  end
+
+  private
+
+  def open_queues
+    @reply_queue = @channel.queue(UUIDTools::UUID.timestamp_create.to_s)
+    @reply_exchange = @channel.direct(UUIDTools::UUID.timestamp_create.to_s)
+    @reply_queue.bind(@reply_exchange)
+    @file_queue = @channel.queue(UUIDTools::UUID.timestamp_create.to_s)
+  end
+
+  def cleanup_queues
+    @reply_queue.delete
+    @file_queue.delete
+  end
+
+  def cancel_message
+    Yajl::Encoder.encode({:action => "cancel_job"})
+  end
+end

data/lib/gorgon/pipe_manager.rb

@@ -0,0 +1,55 @@
+module PipeManager
+  private
+
+  def pipe_fork_worker
+    pid = fork do
+      bind_to_fifos
+      worker = Worker.build(@config)
+      worker.work
+      exit
+    end
+
+    fifo_in, fifo_out, fifo_err = wait_for_fifos pid
+
+    pipe_in = File.open(fifo_in, "w")
+    pipe_out = File.open(fifo_out)
+    pipe_err = File.open(fifo_err)
+
+    return pid, pipe_in, pipe_out, pipe_err
+  end
+
+  def pipe_file pid, stream
+    "#{pid}_#{stream}.pipe"
+  end
+
+  def bind_to_fifos
+    fifo_in = pipe_file $$, "in"
+    fifo_out = pipe_file $$, "out"
+    fifo_err = pipe_file $$, "err"
+
+    system("mkfifo '#{fifo_in}'")
+    system("mkfifo '#{fifo_out}'")
+    system("mkfifo '#{fifo_err}'")
+
+    @@old_in = $stdin
+    $stdin = File.open(fifo_in)
+
+    @@old_out = $stdout
+    $stdout = File.open(fifo_out, "w")
+
+    @@old_err = $stderr
+    $stderr = File.open(fifo_err, "w")
+  end
+
+  def wait_for_fifos pid
+    fifo_in = pipe_file pid, "in"
+    fifo_out = pipe_file pid, "out"
+    fifo_err = pipe_file pid, "err"
+
+    while !File.exist?(fifo_in) || !File.exist?(fifo_out) || !File.exist?(fifo_err)  do
+      sleep 0.01
+    end
+
+    return fifo_in, fifo_out, fifo_err
+  end
+end

data/lib/gorgon/progress_bar_view.rb

@@ -0,0 +1,121 @@
+require 'ruby-progressbar'
+require 'colorize'
+
+MAX_LENGTH = 200
+LOADING_MSG = "Loading environment and workers..."
+RUNNING_MSG = "Running files:"
+LEGEND_MSG = "Legend:\nF - failure files count\nH - number of hosts that have run files\nW - number of workers running files"
+
+FILENAME_COLOR = :light_cyan
+HOST_COLOR = :light_blue
+
+class ProgressBarView
+  def initialize job_state
+    @job_state = job_state
+    @job_state.add_observer(self)
+  end
+
+  def show
+    print LOADING_MSG
+  end
+
+  def update payload={}
+    create_progress_bar_if_started_job_running
+
+    return if @progress_bar.nil? || @finished
+
+    failed_files_count = @job_state.failed_files_count
+
+    @progress_bar.title="F: #{failed_files_count} H: #{@job_state.total_running_hosts} W: #{@job_state.total_running_workers}"
+    if failed_files_count > 0
+      @progress_bar.format(format(bar: :red, title: :default))
+    end
+
+    @progress_bar.progress = @job_state.finished_files_count
+
+    if @job_state.is_job_complete? || @job_state.is_job_cancelled?
+      @finished = true
+      print_summary
+    end
+  end
+
+  def create_progress_bar_if_started_job_running
+    if @progress_bar.nil? && @job_state.state == :running
+      print "\r#{' ' * (LOADING_MSG.length)}\r"
+      puts LEGEND_MSG
+      @progress_bar = ProgressBar.create(:total => @job_state.total_files,
+                                         :length => [terminal_size[0], MAX_LENGTH].min,
+                                         :format => format(bar: :green, title: :white));
+    end
+  end
+
+private
+  def format colors
+    # TODO: decide what bar to use
+    #    bar = "%b>%i".colorize(colors[:bar])
+    bar = "%w>%i".colorize(colors[:bar])
+    title = "%t".colorize(colors[:title])
+
+    "%e [#{bar}] %c/%C | #{title}"
+  end
+
+  def terminal_size
+    `stty size`.split.map { |x| x.to_i }.reverse
+  end
+
+  def print_summary
+    print_failed_tests
+    print_running_files
+    #TODO: print other stats: time, total file, total failures, etc
+  end
+
+  def print_failed_tests
+    @job_state.each_failed_test do |test|
+      puts "\n" + ('*' * 80).magenta #light_red
+      puts("File '#{test[:filename].colorize(FILENAME_COLOR)}' failed/crashed at " \
+           + "'#{test[:hostname].colorize(HOST_COLOR)}'\n")
+      msg = build_fail_message test[:failures]
+      puts "#{msg}\n"
+    end
+  end
+
+  def build_fail_message failures
+    result = []
+    failures.each do |failure|
+      if failure.is_a?(Hash)
+        result << build_fail_message_from_hash(failure)
+      else
+        result << build_fail_message_from_string(failure)
+      end
+    end
+
+    result.join("\n")
+  end
+
+  def print_running_files
+    title = "Unfinished files".yellow
+    puts "\n#{title} - The following files were still running:" if @job_state.total_running_workers > 0
+
+    @job_state.each_running_file do |hostname, filename|
+      filename_str = filename.dup.colorize(FILENAME_COLOR)
+      hostname_str = hostname.dup.colorize(HOST_COLOR)
+      puts "\t#{filename_str} at '#{hostname_str}'"
+    end
+  end
+
+  def build_fail_message_from_string failure
+    result = failure.gsub(/^Error:/, "Error:".yellow)
+    result.gsub!(/^Failure:/, "Failure:".red)
+    result
+  end
+
+  def build_fail_message_from_hash failure
+    result = "#{'Test name'.yellow}: #{failure[:test_name]}"
+    result << "\n#{'Message:'.yellow} \n#{failure[:message]}" if failure[:message]
+    if failure[:location]
+      result << "\n#{'In:'.yellow} \n\t"
+      result << failure[:location].join("\n\t")
+    end
+    result
+  end
+end