gorgon 0.11.0 → 0.11.1

data/lib/gorgon/listener.rb

@@ -16,185 +16,176 @@ require "open4"
 require "tmpdir"
 require "socket"
 
-class Listener
-  include Configuration
-  include GLogger
-  include CrashReporter
-
-  def initialize
-    @listener_config_filename = Dir.pwd + "/gorgon_listener.json"
-    initialize_logger configuration[:log_file]
-
-    log "Listener #{Gorgon::VERSION} initializing"
-    connect
-    initialize_personal_job_queue
-    announce_readiness_to_originators
-  end
-
-  def listen
-    at_exit_hook
-    log "Waiting for jobs..."
-    while true
-      sleep 2 unless poll
+module Gorgon
+  class Listener
+    include Configuration
+    include GLogger
+    include CrashReporter
+
+    def initialize
+      @listener_config_filename = Dir.pwd + "/gorgon_listener.json"
+      initialize_logger configuration[:log_file]
+
+      log "Listener #{Gorgon::VERSION} initializing"
+      connect
+      initialize_personal_job_queue
+      announce_readiness_to_originators
     end
-  end
-
-  def connect
-    @bunny = GorgonBunny.new(connection_information)
-    @bunny.start
-  end
 
-  def initialize_personal_job_queue
-    @job_queue = @bunny.queue("job_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
-    exchange = @bunny.exchange(job_exchange_name, :type => :fanout)
-    @job_queue.bind(exchange)
-  end
+    def listen
+      at_exit_hook
+      log "Waiting for jobs..."
+      while true
+        sleep 2 unless poll
+      end
+    end
 
-  def announce_readiness_to_originators
-    exchange = @bunny.exchange(originator_exchange_name, :type => :fanout)
-    data = {:listener_queue_name => @job_queue.name}
-    exchange.publish(Yajl::Encoder.encode(data))
-  end
+    def connect
+      @bunny = GorgonBunny.new(connection_information)
+      @bunny.start
+    end
 
-  def poll
-    message = @job_queue.pop
-    return false if message == [nil, nil, nil]
-    log "Received: #{message}"
+    def initialize_personal_job_queue
+      @job_queue = @bunny.queue("job_queue_" + UUIDTools::UUID.timestamp_create.to_s, :auto_delete => true)
+      exchange = @bunny.exchange(job_exchange_name, :type => :fanout)
+      @job_queue.bind(exchange)
+    end
 
-    payload = message[2]
+    def announce_readiness_to_originators
+      exchange = @bunny.exchange(originator_exchange_name, :type => :fanout)
+      data = {:listener_queue_name => @job_queue.name}
+      exchange.publish(Yajl::Encoder.encode(data))
+    end
 
-    handle_request payload
+    def poll
+      message = @job_queue.pop
+      return false if message == [nil, nil, nil]
+      log "Received: #{message}"
 
-    log "Waiting for more jobs..."
-    return true
-  end
+      payload = message[2]
 
-  def handle_request json_payload
-    payload = Yajl::Parser.new(:symbolize_keys => true).parse(json_payload)
+      handle_request payload
 
-    case payload[:type]
-    when "job_definition"
-      run_job(payload)
-    when "ping"
-      respond_to_ping payload[:reply_exchange_name]
-    when "gem_command"
-      GemCommandHandler.new(@bunny).handle payload, configuration
+      log "Waiting for more jobs..."
+      return true
     end
-  end
-
-  def run_job(payload)
-    @job_definition = JobDefinition.new(payload)
-    @reply_exchange = @bunny.exchange(@job_definition.reply_exchange_name, :auto_delete => true)
-
-    copy_source_tree(@job_definition.sync)
 
-    if !@syncer.success? || !run_after_sync
-      clean_up
-      return
+    def handle_request json_payload
+      payload = Yajl::Parser.new(:symbolize_keys => true).parse(json_payload)
+
+      case payload[:type]
+      when "job_definition"
+        run_job(payload)
+      when "ping"
+        respond_to_ping payload[:reply_exchange_name]
+      when "gem_command"
+        GemCommandHandler.new(@bunny).handle payload, configuration
+      end
     end
 
-    fork_worker_manager
+    def run_job(payload)
+      @job_definition = JobDefinition.new(payload)
+      @reply_exchange = @bunny.exchange(@job_definition.reply_exchange_name, :auto_delete => true)
 
-    clean_up
-  end
+      syncer = SourceTreeSyncer.new(@job_definition.sync)
+      syncer.pull do |execution_context|
+        if execution_context.success
+          log "Command '#{execution_context.command}' completed successfully."
+          fork_worker_manager if run_after_sync?
+        else
 
-  def at_exit_hook
-    at_exit { log "Listener will exit!"}
-  end
+          log_and_send_crash_message(execution_context.command, execution_context.output, execution_context.errors)
+        end
+      end
+    end
 
-  private
-
-  def run_after_sync
-    log "Running after_sync callback..."
-    begin
-      callback_handler.after_sync
-    rescue Exception => e
-      log_error "Exception raised when running after_sync callback_handler. Please, check your script in #{@job_definition.callbacks[:after_sync]}:"
-      log_error e.message
-      log_error "\n" + e.backtrace.join("\n")
-
-      reply = {:type => :exception,
-        :hostname => Socket.gethostname,
-        :message => "after_sync callback failed. Please, check your script in #{@job_definition.callbacks[:after_sync]}. Message: #{e.message}",
-        :backtrace => e.backtrace.join("\n")
-      }
-      @reply_exchange.publish(Yajl::Encoder.encode(reply))
-      return false
-    end
-    true
-  end
+    def at_exit_hook
+      at_exit { log "Listener will exit!"}
+    end
 
-  def callback_handler
-    @callback_handler ||= CallbackHandler.new(@job_definition.callbacks)
-  end
+    private
+
+    def run_after_sync?
+      log "Running after_sync callback..."
+      begin
+        callback_handler.after_sync
+      rescue Exception => e
+        log_error "Exception raised when running after_sync callback_handler. Please, check your script in #{@job_definition.callbacks[:after_sync]}:"
+        log_error e.message
+        log_error "\n" + e.backtrace.join("\n")
+
+        reply = {:type => :exception,
+                 :hostname => Socket.gethostname,
+                 :message => "after_sync callback failed. Please, check your script in #{@job_definition.callbacks[:after_sync]}. Message: #{e.message}",
+                 :backtrace => e.backtrace.join("\n")
+        }
+        @reply_exchange.publish(Yajl::Encoder.encode(reply))
+        return false
+      end
+      true
+    end
 
-  def copy_source_tree(sync_configuration)
-    log "Downloading source tree to temp directory..."
-    @syncer = SourceTreeSyncer.new sync_configuration
-    @syncer.sync
-    if @syncer.success?
-      log "Command '#{@syncer.sys_command}' completed successfully."
-    else
-      send_crash_message @reply_exchange, @syncer.output, @syncer.errors
-      log_error "Command '#{@syncer.sys_command}' failed!"
-      log_error "Stdout:\n#{@syncer.output}"
-      log_error "Stderr:\n#{@syncer.errors}"
+    def callback_handler
+      @callback_handler ||= Gorgon::CallbackHandler.new(@job_definition.callbacks)
     end
-  end
 
-  def clean_up
-    @syncer.remove_temp_dir
-  end
+    def log_and_send_crash_message(command, output, errors)
+      send_crash_message @reply_exchange, output, errors
+      log_error "Command '#{command}' failed!"
+      log_error "Stdout:\n#{output}"
+      log_error "Stderr:\n#{errors}"
+    end
 
-  ERROR_FOOTER_TEXT = "\n***** See #{WorkerManager::STDERR_FILE} and #{WorkerManager::STDOUT_FILE} at '#{Socket.gethostname}' for complete output *****\n"
-  def fork_worker_manager
-    log "Forking Worker Manager..."
-    ENV["GORGON_CONFIG_PATH"] = @listener_config_filename
+    ERROR_FOOTER_TEXT = "\n***** See #{WorkerManager::STDERR_FILE} and #{WorkerManager::STDOUT_FILE} at '#{Socket.gethostname}' for complete output *****\n"
+    def fork_worker_manager
+      log "Forking Worker Manager..."
+      ENV["GORGON_CONFIG_PATH"] = @listener_config_filename
 
-    pid, stdin = Open4::popen4 "gorgon manage_workers"
-    stdin.write(@job_definition.to_json)
-    stdin.close
+      pid, stdin = Open4::popen4 "gorgon manage_workers"
+      stdin.write(@job_definition.to_json)
+      stdin.close
 
-    _, status = Process.waitpid2 pid
-    log "Worker Manager #{pid} finished"
+      _, status = Process.waitpid2 pid
+      log "Worker Manager #{pid} finished"
 
-    if status.exitstatus != 0
-      exitstatus = status.exitstatus
-      log_error "Worker Manager #{pid} crashed with exit status #{exitstatus}!"
+      if status.exitstatus != 0
+        exitstatus = status.exitstatus
+        log_error "Worker Manager #{pid} crashed with exit status #{exitstatus}!"
 
-      msg = report_crash @reply_exchange, :out_file => WorkerManager::STDOUT_FILE,
-      :err_file => WorkerManager::STDERR_FILE, :footer_text => ERROR_FOOTER_TEXT
+        msg = report_crash @reply_exchange, :out_file => WorkerManager::STDOUT_FILE,
+          :err_file => WorkerManager::STDERR_FILE, :footer_text => ERROR_FOOTER_TEXT
 
-      log_error "Process output:\n#{msg}"
+        log_error "Process output:\n#{msg}"
+      end
     end
-  end
 
-  def respond_to_ping reply_exchange_name
-    reply = {:type => "ping_response", :hostname => Socket.gethostname,
-      :version => Gorgon::VERSION, :worker_slots => configuration[:worker_slots]}
-    publish_to reply_exchange_name, reply
-  end
+    def respond_to_ping reply_exchange_name
+      reply = {:type => "ping_response", :hostname => Socket.gethostname,
+               :version => Gorgon::VERSION, :worker_slots => configuration[:worker_slots]}
+      publish_to reply_exchange_name, reply
+    end
 
-  def publish_to reply_exchange_name, message
-    reply_exchange = @bunny.exchange(reply_exchange_name, :auto_delete => true)
+    def publish_to reply_exchange_name, message
+      reply_exchange = @bunny.exchange(reply_exchange_name, :auto_delete => true)
 
-    log "Sending #{message}"
-    reply_exchange.publish(Yajl::Encoder.encode(message))
-  end
+      log "Sending #{message}"
+      reply_exchange.publish(Yajl::Encoder.encode(message))
+    end
 
-  def job_exchange_name
-    OriginatorProtocol.job_exchange_name(configuration.fetch(:cluster_id, nil))
-  end
+    def job_exchange_name
+      OriginatorProtocol.job_exchange_name(configuration.fetch(:cluster_id, nil))
+    end
 
-  def originator_exchange_name
-    OriginatorProtocol.originator_exchange_name(configuration.fetch(:cluster_id, nil))
-  end
+    def originator_exchange_name
+      OriginatorProtocol.originator_exchange_name(configuration.fetch(:cluster_id, nil))
+    end
 
-  def connection_information
-    configuration[:connection]
-  end
+    def connection_information
+      configuration[:connection]
+    end
 
-  def configuration
-    @configuration ||= load_configuration_from_file("gorgon_listener.json")
+    def configuration
+      @configuration ||= load_configuration_from_file("gorgon_listener.json")
+    end
   end
 end

data/lib/gorgon/listener_installer.rb

@@ -1,122 +1,123 @@
 require 'etc'
 require 'fileutils'
 
-class ListenerInstaller
-  include Configuration
+module Gorgon
+  class ListenerInstaller
+    include Configuration
 
-  GORGON_DIR=".gorgon"
-  DEFAULT_NO_WORKERS = 4
-  LISTENER_CONFIG_FILE = "gorgon_listener.json"
-  GORGON_INIT_FILE = "/etc/init/gorgon.conf"
+    GORGON_DIR=".gorgon"
+    DEFAULT_NO_WORKERS = 4
+    LISTENER_CONFIG_FILE = "gorgon_listener.json"
+    GORGON_INIT_FILE = "/etc/init/gorgon.conf"
 
-  def self.install
-    ListenerInstaller.new.install
-  end
-
-  def install
-    @configuration = load_configuration_from_file("gorgon.json")
-
-    FileUtils.mkdir_p gorgon_dir_path
-    Dir.chdir gorgon_dir_path
+    def self.install
+      ListenerInstaller.new.install
+    end
 
-    create_listener_config_file
+    def install
+      @configuration = load_configuration_from_file("gorgon.json", merge: "gorgon_secret.json")
 
-    create_gorgon_service
+      FileUtils.mkdir_p gorgon_dir_path
+      Dir.chdir gorgon_dir_path
 
-    start_gorgon_daemon
-  end
+      create_listener_config_file
 
-  private
+      create_gorgon_service
 
-  def gorgon_dir_path
-    @gorgon_dir_path ||= "#{Dir.home}/#{GORGON_DIR}"
-  end
+      start_gorgon_daemon
+    end
 
-  def create_listener_config_file
-    worker_slots = worker_slots_prompt
-    puts "Using #{worker_slots} worker slots"
-    amqp_host = @configuration[:connection][:host]
-    puts "Amqp host is '#{amqp_host}'"
+    private
 
-    puts "Creating #{LISTENER_CONFIG_FILE} in #{Dir.pwd}"
-    File.open(LISTENER_CONFIG_FILE, 'w') do |f|
-      f.write listener_config(amqp_host, worker_slots)
+    def gorgon_dir_path
+      @gorgon_dir_path ||= "#{Dir.home}/#{GORGON_DIR}"
     end
-  end
 
-  def create_gorgon_service
-    params = {}
-    params[:username] = Etc.getlogin
-    params[:rvm_bin_path] = rvm_bin_path
-    params[:gemset] = get_current_gemset
+    def create_listener_config_file
+      worker_slots = worker_slots_prompt
+      puts "Using #{worker_slots} worker slots"
+      amqp_host = @configuration[:connection][:host]
+      puts "Amqp host is '#{amqp_host}'"
 
-    tmp_gorgon_conf = '/tmp/gorgon.conf'
-    File.open(tmp_gorgon_conf, 'w') do |f|
-      f.write gorgon_conf(params)
+      puts "Creating #{LISTENER_CONFIG_FILE} in #{Dir.pwd}"
+      File.open(LISTENER_CONFIG_FILE, 'w') do |f|
+        f.write listener_config(amqp_host, worker_slots)
+      end
     end
 
-    puts "Creating '#{GORGON_INIT_FILE}'"
-
-    system("sudo cp /tmp/gorgon.conf #{GORGON_INIT_FILE}")
-  end
+    def create_gorgon_service
+      params = {}
+      params[:username] = Etc.getlogin
+      params[:rvm_bin_path] = rvm_bin_path
+      params[:gemset] = get_current_gemset
 
-  def rvm_bin_path
-    cmd = 'which rvm'
-    path = get_shell_cmd_output cmd, "Error getting rvm path. Make sure '#{cmd}' works"
-    puts "Using '#{path}'"
-    path
-  end
+      tmp_gorgon_conf = '/tmp/gorgon.conf'
+      File.open(tmp_gorgon_conf, 'w') do |f|
+        f.write gorgon_conf(params)
+      end
 
-  def get_current_gemset
-    return @gemset unless @gemset.nil?
+      puts "Creating '#{GORGON_INIT_FILE}'"
 
-    cmd = 'rvm current'
-    @gemset = get_shell_cmd_output cmd, "Error getting current gem. Make sure '#{cmd}' works"
-    puts "Using gemset '#{@gemset}'."
-    @gemset
-  end
+      system("sudo cp /tmp/gorgon.conf #{GORGON_INIT_FILE}")
+    end
 
-  def get_shell_cmd_output cmd, error_message
-    result = `#{cmd}`.strip
-    if $?.exitstatus != 0
-      $stderr.puts "#{error_message}"
-      $stderr.puts "Aborting installation"
-      exit
+    def rvm_bin_path
+      cmd = 'which rvm'
+      path = get_shell_cmd_output cmd, "Error getting rvm path. Make sure '#{cmd}' works"
+      puts "Using '#{path}'"
+      path
     end
-    result
-  end
 
-  def start_gorgon_daemon
-    system("sudo start gorgon")
-  end
+    def get_current_gemset
+      return @gemset unless @gemset.nil?
 
-  def gem_available?(name)
-    Gem::Specification.find_by_name(name)
-  rescue Gem::LoadError
-    false
-  rescue
-    Gem.available?(name)
-  end
+      cmd = 'rvm current'
+      @gemset = get_shell_cmd_output cmd, "Error getting current gem. Make sure '#{cmd}' works"
+      puts "Using gemset '#{@gemset}'."
+      @gemset
+    end
 
-  def worker_slots_prompt
-    begin
-      puts "Number of worker slots (default #{DEFAULT_NO_WORKERS})?"
-      input = $stdin.gets.chomp
-      if input == ""
-        worker_slots = DEFAULT_NO_WORKERS
-        break
+    def get_shell_cmd_output cmd, error_message
+      result = `#{cmd}`.strip
+      if $?.exitstatus != 0
+        $stderr.puts "#{error_message}"
+        $stderr.puts "Aborting installation"
+        exit
       end
+      result
+    end
 
-      worker_slots = input.to_i
-      if worker_slots <= 0 || worker_slots >= 20
-        puts "Please, enter a valid number between 0 and 19"
-      end
-    end while worker_slots <= 0 || worker_slots >= 20
-    worker_slots
-  end
+    def start_gorgon_daemon
+      system("sudo start gorgon")
+    end
 
-  def listener_config amqp_host, worker_slots
-    <<-CONFIG
+    def gem_available?(name)
+      Gem::Specification.find_by_name(name)
+    rescue Gem::LoadError
+      false
+    rescue
+      Gem.available?(name)
+    end
+
+    def worker_slots_prompt
+      begin
+        puts "Number of worker slots (default #{DEFAULT_NO_WORKERS})?"
+        input = $stdin.gets.chomp
+        if input == ""
+          worker_slots = DEFAULT_NO_WORKERS
+          break
+        end
+
+        worker_slots = input.to_i
+        if worker_slots <= 0 || worker_slots >= 20
+          puts "Please, enter a valid number between 0 and 19"
+        end
+      end while worker_slots <= 0 || worker_slots >= 20
+      worker_slots
+    end
+
+    def listener_config amqp_host, worker_slots
+      <<-CONFIG
 {
   "connection": {
     "host": "#{amqp_host}"
@@ -125,11 +126,11 @@ class ListenerInstaller
   "worker_slots": #{worker_slots},
   "log_file": "/tmp/gorgon-remote.log"
 }
-CONFIG
-  end
+      CONFIG
+    end
 
-  def gorgon_conf params
-    <<-CONF_FILE
+    def gorgon_conf params
+      <<-CONF_FILE
 description "Start gorgon listener"
 
 start on filesystem or runlevel [2345]
@@ -148,6 +149,7 @@ EOF
 end script
 
 exec su - #{params[:username]} -c 'cd #{gorgon_dir_path} && #{params[:rvm_bin_path]} #{params[:gemset]} do gorgon listen >> /var/log/gorgon/gorgon.log 2>&1'
-CONF_FILE
+      CONF_FILE
+    end
   end
 end