hive-runner 2.0.0

data/lib/hive/diagnostic_runner.rb

@@ -0,0 +1,32 @@
+module Hive
+  class DiagnosticRunner
+    attr_accessor :diagnostics, :options
+
+    def initialize(options, diagnostics_config, platform)
+      @options = options
+      @platform = platform
+      @diagnostics = self.initialize_diagnostics(diagnostics_config[@platform]) if diagnostics_config.has_key?(@platform)
+    end
+
+    def initialize_diagnostics(diagnostics_config)
+      if diagnostics_config
+        @diagnostics = diagnostics_config.collect do |component, config|
+          Hive.logger.info("Initializing #{component.capitalize} component for #{@platform.capitalize} diagnostic")
+          require "hive/diagnostic/#{@platform}/#{component}"
+          Object.const_get('Hive').const_get('Diagnostic').const_get(@platform.capitalize).const_get(component.capitalize).new(config, @options)
+        end
+      else
+        Hive.logger.info("No diagnostic specified for #{@platform}")
+      end
+    end 
+
+    def run
+      results = @diagnostics.collect do |diagnostic|
+        diagnostic.run
+      end
+     
+      failures = results.select { |r| r.failed? }
+      failures.count == 0
+    end
+  end
+end

data/lib/hive/execution_script.rb

@@ -0,0 +1,121 @@
+require 'timeout'
+
+module Hive
+  class ExecutionScript
+    def initialize(config)
+      @path = config[:file_system].executed_script_path
+      @log_path = config[:file_system].logs_path
+      @log = config[:log]
+      @keep_running = config[:keep_running]
+      @log.debug "Creating execution script with path=#{@path}"
+      @env = {
+        'HIVE_SCHEDULER' => Hive.config.network.scheduler,
+        'HIVE_WORKING_DIRECTORY' => config[:file_system].testbed_path
+      }
+      @env_unset = [
+        'BUNDLE_GEMFILE',
+        'BUNDLE_BIN_PATH',
+        'GEM_PATH',
+        'RUBYOPT',
+        'rvm_'
+      ]
+      # Environment variables that should not be made visible in the execution
+      # script uploaded with the results
+      @env_secure = {
+        'HIVE_CERT' => Hive.config.network.cert
+      }
+      @script_lines = []
+    end
+
+    def prepend_bash_cmd(shell_command)
+      @log.debug "bash.rb - Prepending bash command to #{@path} script: " + shell_command
+      @script_lines = ([] << shell_command << @script_lines).flatten
+    end
+
+    def append_bash_cmd(shell_command)
+      @log.debug "bash.rb - Appending bash command to #{@path} script: " + shell_command
+      @script_lines << shell_command
+    end
+
+    def set_env(var, value)
+      @env[var] = value
+
+      # TODO What if the element appears multiple times?
+      if (i = @env_unset.index(var))
+        @env_unset.delete(i)
+      end
+
+      ## In Ruby 2, replace the above 'if' block with ...
+      #@env_unset.remove(var)
+    end
+
+    def unset_env(var)
+      @env.delete(var)
+      @env_unset << var
+    end
+    
+    def helper_path
+      scripts_dir = File.expand_path(File.dirname(__FILE__) + "../../../scripts/")
+      File.join(scripts_dir, 'hive-script-helper.sh')
+    end
+
+    def run
+      @log.info 'bash.rb - Writing script out to file'
+      File.open(@path, 'w') do |f|
+        f.write("#!/bin/bash --login\n")
+        f.write(". #{helper_path}\n")
+        f.write("# Set environment\n")
+        @env.each do |key, value|
+          # An escaped ' in a single quoted string in bash looks like '"'"'
+          if value.kind_of?(Array)
+            f.write("export #{key}=(" + value.collect { |v| "'#{v.to_s.gsub("'", '\'"\'"\'')}'" }.join(' ') + ")\n" )
+          else
+            f.write("export #{key}='#{value.to_s.gsub("'", '\'"\'"\'')}'\n")
+          end
+        end
+        @env_unset.each do |var|
+          f.write("unset #{var}\n")
+        end
+        f.write("cd $HIVE_WORKING_DIRECTORY")
+        f.write("\n# Test execution\n")
+        f.write(@script_lines.join("\n"))
+      end
+      File.chmod(0700, @path)
+
+      pid = Process.spawn @env_secure, "#{@path}", pgroup: true, in: '/dev/null', out: "#{@log_path}/stdout.log", err: "#{@log_path}/stderr.log"
+      @pgid = Process.getpgid(pid)
+
+      exit_value = nil
+      running = true
+      while running
+        begin
+          Timeout.timeout(30) do
+            Process.wait pid
+            exit_value = $?.exitstatus
+            running = false
+          end
+        rescue Timeout::Error
+          Process.kill(-9, @pgid) if ! ( @keep_running.nil? || @keep_running.call )
+          # Do something. Eg, upload log files.
+        end
+      end
+
+      # Kill off anything that is still running
+      terminate
+
+      # Return exit value of the script
+      exit_value
+    end
+
+    def terminate
+      if @pgid
+        begin
+          Process.kill(-9, @pgid)
+        rescue => e
+          @log.warn e
+        end
+        @pgid = nil
+      end
+    end
+  end
+end

data/lib/hive/file_system.rb

@@ -0,0 +1,103 @@
+require "fileutils"
+
+module Hive
+  class FileSystem
+    def initialize(job_id, home_directory, log)
+      @job_id = job_id
+      @home_directory = home_directory
+      @log = log
+      @log.debug "Creating job paths with id=#{@job_id} and home=#{@home_directory}"
+      make_directory(home_path)
+      make_directory(results_path)
+      make_directory(logs_path)
+      make_directory(testbed_path)
+    end
+
+    def home_path
+      @home_path ||= "#{@home_directory}/#{@job_id.to_s}"
+    end
+
+    def results_path
+      @results_path ||= "#{home_path}/results"
+    end
+
+    def logs_path
+      @logs_path ||= "#{home_path}/logs"
+    end
+
+    def testbed_path
+      @testbed_path ||= "#{home_path}/test_code"
+    end
+
+    def executed_script_path
+      @bash_script_path ||= "#{testbed_path}/executed_script.sh"
+    end
+
+    # Copy useful stuff into the results directory
+    def finalise_results_directory
+      copy_file(executed_script_path, "#{results_path}/executed_script.sh")
+    end
+
+    def fetch_build(build_url, destination_path)
+      if !fetch_build_with_curl(build_url, destination_path)
+        @log.info( "Initial build fetch failed -- trying again shortly")
+        sleep 5
+        if !fetch_build_with_curl(build_url, destination_path)
+          raise "Build could not be downloaded"
+        end
+      end
+    end
+
+    def fetch_build_with_curl(build_url, destination_path)
+      cert_path     = Hive.config.network['cert']
+      cabundle_path = Hive.config.network['cafile']
+      base_url      = Hive.config.network['scheduler']
+      apk_url       = base_url + '/' + build_url
+      curl_line     = "curl -L -m 60 #{apk_url} --cert #{cert_path} --cacert #{cabundle_path} --retry 3 -o #{destination_path}"
+
+      @log.info("Fetching build from hive-scheduler: #{curl_line}")
+      @log.debug("CURL line: #{curl_line}")
+      response = `#{curl_line}`
+      if $? != 0
+        @log.info("Curl error #{$?}: #{response.to_s}")
+        false
+        Hive::Messages
+      else
+        @log.info("Curl seems happy, checking integrity of downloaded file")
+        check_build_integrity( destination_path )
+      end
+    end
+
+    def check_build_integrity( destination_path )
+      output = `file #{destination_path}`
+      if output =~ /zip/
+        result = `zip -T #{destination_path}`
+        @log.info(result)
+        $? == 0
+      else
+        true
+      end
+    end
+
+    private
+
+    def copy_file(src, dest)
+      begin
+        FileUtils.cp(src, dest)
+        @log.debug("Copied file #{src} -> #{dest}")
+      rescue => e
+        @log.error(e.message)
+      end
+    end
+
+    def make_directory(directory)
+      begin
+        FileUtils.rm_r(directory) if File.directory?(directory)
+        FileUtils.mkdir_p(directory)
+        @log.debug("Created directory: #{directory}")
+      rescue => e
+        @log.fatal(e.message)
+      end
+    end
+  end
+end

data/lib/hive/log.rb

@@ -0,0 +1,60 @@
+require 'mono_logger'
+
+module Hive
+  # Hive logging
+  # Allow logging to be written to multiple locations.
+  class Log
+    # Create the logger:
+    #
+    #   # No log files will be written
+    #   log = Hive::Log.new()
+    #   # Write log files to standard out and a log file
+    #   log = Hive::Log.new( [
+    #                           {
+    #                             stream: 'Filename.log',
+    #                             level: 'DEBUG'
+    #                           },
+    #                           {
+    #                             stream: STDOUT,
+    #                             level: 'INFO'
+    #                           },
+    #                         ] )
+    def initialize(args = [])
+      @loggers = {}
+      args.each do |l|
+        add_logger(l[:stream], l[:level])
+      end
+    end
+
+    # Add a new log location:
+    #
+    #   # INFO level log to 'Filename.log'
+    #   log.add_logger( 'Filename.log', 'INFO' )
+    #   # DEBUG level log to standard output
+    #   log.add_logger( STDOUT, 'DEBUG' )
+    def add_logger(stream, level)
+      log = MonoLogger.new(stream)
+      log.formatter = proc do |severity, datetime, _progname, msg|
+        "#{severity[0, 1]} #{datetime.strftime('%Y-%m-%d %H:%M:%S')}: #{msg}\n"
+      end
+      log.level = MonoLogger.const_get(level)
+      @loggers[stream] = log
+    end
+
+    # Stop a log stream:
+    #
+    #   # Stop the log to standard output
+    #   log.stop_logger( STDOUT )
+    #   # Stop the log to 'Filename.log'
+    #   log.stop_logger( 'Filename.log' )
+    def stop_logger(stream)
+      @loggers.delete(stream)
+    end
+
+    MonoLogger::Severity.constants.each do |level|
+      define_method(level.downcase) do |*args|
+        @loggers.each { |_s, l| l.send(level.downcase, *args) }
+      end
+    end
+  end
+end

data/lib/hive/port_allocator.rb

@@ -0,0 +1,79 @@
+module Hive
+  class PortAllocator
+    class NoPortsAvailable < StandardError
+    end
+
+    # Create a port allocator
+    #
+    # For ports in the range 4000-5000
+    #   Hive::PortAllocator.new(minimum: 4000, maximum: 5000)
+    #
+    # For ports 6000, 6050 and 7433
+    #   Hive::PortAllocator.new(ports: [6000, 6050, 7433])
+    #
+    def initialize(config)
+      @allocated_ports = []
+      if config.has_key?(:minimum) and config.has_key?(:maximum) and config[:minimum] > 0 and config[:minimum] <= config[:maximum]
+        @free_ports = Array(config[:minimum]..config[:maximum])
+      elsif config.has_key?(:ports) and config[:ports].is_a? Array
+        config[:ports].each do |p|
+          raise ArgumentError if ! p.is_a? Integer or p <= 0
+        end
+        @free_ports = config[:ports]
+      else
+        raise ArgumentError
+      end
+    end
+
+    # Allocate a single port in the range
+    def allocate_port
+      if p = @free_ports.pop
+        @allocated_ports << p
+        p
+      else
+        raise NoPortsAvailable
+      end
+    end
+
+    # Relase a single port in the range
+    def release_port(p)
+      @free_ports << p if @allocated_ports.delete(p)
+    end
+
+    # Create a new Hive::PortAllocator instance with a number of ports from
+    # the range
+    def allocate_port_range(n)
+      if n <= @free_ports.length
+        ps = @free_ports.take(n)
+        @free_ports = @free_ports.drop(n)
+        @allocated_ports.concat(ps)
+        PortAllocator.new(ports: ps)
+      else
+        raise NoPortsAvailable
+      end
+    end
+
+    # Release ports that were previously allocated to another
+    # Hive::PortAllocator
+    #
+    # Note, this will fail silently if 'range' contains ports that are not
+    # allocated in the current instance
+    def release_port_range(range)
+      if range.ports - @allocated_ports == []
+        @free_ports.concat(range.ports)
+        @allocated_ports = @allocated_ports - range.ports
+      end
+    end
+
+    # Release all ports
+    def release_all_ports
+      @free_ports.concat(@allocated_ports)
+      @allocated_ports = []
+    end
+
+    # Full list of all ports, either free or allocated
+    def ports
+      [@free_ports, @allocated_ports].flatten
+    end
+  end
+end

data/lib/hive/register.rb

@@ -0,0 +1,120 @@
+require 'hive'
+require 'hive/port_allocator'
+
+module Hive
+  # Central register of devices and workers in the hive
+  class Register
+    attr_reader :controllers
+
+    def initialize
+      @controllers = []
+      @devices = {}
+      @max_devices = 5 # TODO Add to configuration file
+      if Hive.config.ports?
+        @port_allocator = Hive::PortAllocator.new(minimum: Hive.config.ports.minimum, maximum: Hive.config.ports.maximum)
+      else
+        @port_allocator = Hive::PortAllocator.new(ports: [])
+      end
+    end
+
+    def devices
+      list = []
+      @devices.each do |controller, device_list|
+        list.concat(device_list)
+      end
+      list
+    end
+
+    def worker_pids
+      self.devices.collect{ |d| d.worker_pid }.compact
+    end
+
+    def instantiate_controllers(controller_details = Hive.config.controllers)
+      controller_details.each do |type, opts|
+        Hive.logger.info("Adding controller for '#{type}'")
+        require "hive/controller/#{type}"
+        controller = Object.const_get('Hive').const_get('Controller').const_get(type.capitalize).new(opts.to_hash)
+        @controllers << controller
+      end
+      check_controllers
+      @controllers
+    end
+
+    def run
+      loop do
+        Hive.poll
+        housekeeping
+        check_controllers
+        sleep Hive.config.timings.controller_loop_interval
+      end
+    end
+
+    def check_controllers
+      Hive.logger.debug("Devices before update: #{@devices.inspect}")
+      new_device_list = {}
+      @controllers.each do |c|
+        begin
+          new_device_list[c.class] = []
+          @devices[c.class] = [] if ! @devices.has_key?(c.class)
+          Hive.logger.info("Checking controller #{c.class}")
+          c.detect.each do |device|
+            Hive.logger.debug("Found #{device.inspect}")
+            i = @devices[c.class].find_index(device)
+            if i
+              @devices[c.class][i].status = device.status
+              new_device_list[c.class] << @devices[c.class][i]
+            else
+              device.port_allocator = @port_allocator.allocate_port_range(c.port_range_size)
+              new_device_list[c.class] << device
+            end
+          end
+          Hive.logger.debug("new_device_list: #{new_device_list.inspect}")
+
+          # Remove any devices that have not been rediscovered
+          (@devices[c.class] - new_device_list[c.class]).each do |d|
+            @port_allocator.release_port_range(d.port_allocator)
+            d.stop
+            @devices[c.class].delete(d)
+          end
+
+          # Add any new devices
+          (new_device_list[c.class] - @devices[c.class]).each do |d|
+            @devices[c.class] << d
+          end
+          # Check that all known devices have running workers
+          @devices[c.class].each do |d|
+            if d.claimed?
+              d.stop if d.running?
+            else
+              d.start if ! d.running?
+            end
+          end
+        rescue Hive::Controller::DeviceDetectionFailed
+          Hive.logger.warn("Failed to detect devices for #{c.class}")
+        end
+      end
+      Hive.logger.debug("Devices after update: #{@devices.inspect}")
+    end
+
+    def housekeeping
+      clear_workspaces
+    end
+
+    def clear_workspaces
+      candidates = Dir.glob("#{Hive.config.logging.home}/*")
+        .select{ |f|
+          File.directory?(f) \
+          && File.exists?("#{f}/job_info") \
+          && File.read("#{f}/job_info").chomp.to_s =~ /completed/
+        }.sort_by{ |f|
+          File.mtime(f)
+        }.reverse
+      if candidates && candidates.length > Hive.config.logging.homes_to_keep
+        candidates[Hive.config.logging.homes_to_keep..-1].each do |dir|
+          Hive.logger.info("Found (and deleting) #{dir}")
+          FileUtils.rm_rf(dir)
+        end
+      end
+    end
+  end
+end

data/lib/hive/results.rb

@@ -0,0 +1,20 @@
+
+module Hive
+  class Results
+    attr_reader :timestamp
+    def initialize( state, message, data = {})
+      @state = state
+      @message = message
+      @data = data  
+      @timestamp = Time.now
+    end
+  
+    def failed?
+      @state == 'fail'
+    end
+
+    def passed?
+      @state == 'pass'
+    end
+  end
+end

data/lib/hive/worker/shell.rb

@@ -0,0 +1,16 @@
+require 'hive/worker'
+
+module Hive
+  class Worker
+    # The Shell worker
+    class Shell < Worker
+      def initialize(options = {})
+        @devicedb_register = false
+        super
+      end
+
+      def pre_script(job, file_system, script)
+      end
+    end
+  end
+end