cloudscale-benchmark 0.0.2

data/lib/cloudscale/benchmark/model/benchmark/benchmark_run_result_value.rb

@@ -0,0 +1,28 @@
+require "hashable"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Benchmark
+    
+    class BenchmarkRunResultValue
+      include Hashable
+  
+      attr_accessor :name, :description, :arguments, :value, :rawString, :scale, :timestamp
+      
+      def initialize()
+      end
+  
+      def to_h
+        hash = {}
+        instance_variables.each {|var| hash[var.to_s.delete("@")] = instance_variable_get(var) }
+        return hash
+      end
+  
+    end
+    
+  end
+end

data/lib/cloudscale/benchmark/version.rb

@@ -0,0 +1,5 @@
+module Cloudscale
+  module Benchmark
+    VERSION = "0.0.2"
+  end
+end

data/lib/cloudscale/benchmark.rb

@@ -0,0 +1,98 @@
+require "#{File.dirname(__FILE__)}/benchmark/version"
+require "http"
+require "optparse"
+require "singleton"
+require "yaml"
+require "json"
+require "#{File.dirname(__FILE__)}/benchmark/agent/init"
+require "#{File.dirname(__FILE__)}/plugins/plugin"
+require "#{File.dirname(__FILE__)}/plugins/preops/plugin_preop"
+Dir["#{File.dirname(__FILE__)}/plugins/*/*.rb"].each { | f | require f }
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Benchmark
+    options = {}
+    mandatory = Array.new
+      
+    agent_instance = Cloudscale::Benchmark::Init.new
+    agent_instance_id = agent_instance.agent_instance_id()
+    
+    Plugins::PluginPreop.instance.init
+    
+    parser = OptionParser.new do | parser |
+      
+      parser.banner = "Usage: benchmark COMMAND [OPTIONS]"
+      parser.separator  ""
+      parser.separator  "Commands"
+      parser.separator  "     start:  Start cloudscale benchmark agent"
+      parser.separator  "     stop:   Stop cloudscale benchmark agent"
+      parser.separator  ""    
+      parser.separator  "Options"
+  
+      parser.on("-v", "--version", "Show version information about this program and quit.") do
+        puts "Cloudscale Benchmarking " + VERSION
+        exit
+      end
+      
+      parser.separator "\n"
+      parser.separator "Plugin specific Settings"
+      
+      Plugins::PluginPreop.instance.options.each do | key, value |
+        if (value[:required])
+          mandatory.push(key)
+        end
+        parser.on(value[:argument][1,2] + " n", value[:argument] + "=n", value[:description]) do | value |
+          Plugins::PluginPreop.instance.set_option_value(key, value)
+        end
+      end
+    end
+    
+    begin
+      parser.parse!
+      missing = mandatory.select{ | param |
+        Plugins::PluginPreop.instance.get_option_value(param).nil?
+      }
+      if not missing.empty?
+        puts "\n"
+        puts "Error during execution: "
+        puts "     Missing options: #{missing.join(', ')}"
+        puts "\n"
+        puts parser
+        exit
+      else
+        Plugins::PluginPreop.instance.save_options
+      end
+    rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+      puts $!.to_s
+      puts parser
+      exit
+    end           
+    
+    case ARGV[0]
+      when "start"
+        plugins = Set.new
+        
+        Plugins::Plugin.plugins.each do | plugin_class |
+          plugin = plugin_class.new
+          plugins << plugin
+        end
+        
+        plugins.each do | plugin |
+          if plugin.respond_to?('run')
+            plugin.run(agent_instance_id)
+          end
+        end
+        
+      when "stop"
+        puts "call stop on options #{options.inspect}"
+      else
+        puts parser
+    end
+
+  end
+end

data/lib/cloudscale/constants/agent_instance_store.rb

@@ -0,0 +1,42 @@
+require "yaml/store"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+  module Constants
+    class AgentInstance
+  
+      @@AGENT_INSTANCE_STORE_PATH = "#{File.dirname(__FILE__)}/../store/agent/agent_instance.yml"
+  
+      def self.create(hash)
+        settings = YAML::Store.new(@@AGENT_INSTANCE_STORE_PATH)
+  
+        settings.transaction {
+          hash.each { | key, value |
+            settings[key] = value
+          }
+        }
+      end
+  
+      def self.load
+        begin
+          YAML.load(File.read(@@AGENT_INSTANCE_STORE_PATH))
+        rescue
+          return nil
+        end
+      end
+      
+      
+      def self.remove
+        file = File.read(@@AGENT_INSTANCE_STORE_PATH)
+        if (file)
+          File.delete(@@AGENT_INSTANCE_STORE_PATH)
+        end
+      end
+  
+    end
+  end  
+end

data/lib/cloudscale/constants/agent_store.rb

@@ -0,0 +1,25 @@
+require "yaml/store"
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+#
+##
+module Cloudscale
+
+  module Constants
+    class Agent
+      
+      @@AGENT_STORE = "#{File.dirname(__FILE__)}/../store/agent/agent.yml"
+      
+      def self.load
+        begin
+          YAML.load(File.read(@@AGENT_STORE))
+        rescue
+          return nil
+        end
+      end    
+    end
+  end
+  
+end

data/lib/cloudscale/plugins/phoronix/phoronix_bulk_execution.rb

@@ -0,0 +1,259 @@
+require "popen4"
+require "#{File.dirname(__FILE__)}/../../benchmark/model/benchmark/benchmark_run"
+require "#{File.dirname(__FILE__)}/../../benchmark/model/benchmark/benchmark_run_result"
+require "#{File.dirname(__FILE__)}/../../benchmark/model/benchmark/benchmark_run_result_value"
+require "#{File.dirname(__FILE__)}/../../plugins/preops/plugin_preop"
+require "#{File.dirname(__FILE__)}/../../plugins/plugin"
+require "#{File.dirname(__FILE__)}/../../rest/rest_client"
+require "xmlsimple"
+require "socket"
+
+
+##
+#
+# @author Johannes Hiemer, cloudscale.
+# @author Joerg Gottschlich, cloudscale.
+#
+##
+module Cloudscale
+  module Plugins
+    class PhoronixBulkExecutor < Plugins::Plugin
+      attr_reader :none, :homeDir, :fileSeparator, :development, :numberOfRuns, :properties, :nanoSeconds
+      attr_accessor :benchmarkRun, :benchmarkRunResult, :benchmarkTest, :environment
+              
+      def log
+        @log = Logger.new(STDOUT)
+      end
+      
+      def initialize
+        super
+        @options = Plugins::PluginPreop.instance.options
+        @settings = load_hash()
+        @homeDir = Dir.home
+        @fileSeparator = File::Separator
+        @nanoSeconds = 1000000000
+        @environment = []
+        # select first non-loopback ip address
+        @ip_address = Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }.ip_address
+        @hostname = Socket.gethostname
+      end
+      
+      def load_hash()
+        begin 
+          YAML.load(File.read("#{File.dirname(__FILE__)}/../../store/settings/phoronix.yml"))
+        rescue
+          return nil
+        end
+      end
+      
+      def resultDir
+        @homeDir + @fileSeparator + ".phoronix-test-suite" + @fileSeparator + "test-results"
+      end
+      
+      def lockFile
+        @homeDir + @fileSeparator + ".phoronix-test-suite" + @fileSeparator + "run_lock"
+      end
+      
+      def compileParams(benchmarkCommand, testLabel)
+        benchmarkCommand + " " + testLabel
+      end
+      
+      def testLabel(benchmarkTest)
+        "pts/" + benchmarkTest["identifier"] + "-" + benchmarkTest["version"]
+      end
+      
+      def isRunning
+        File.exists?(lockFile)
+      end
+  
+      def run(agentInstanceId)
+        
+        begin  # main loop
+          
+          if (isRunning())
+            log.info("There are currently test cases running on this machine. Don't run two instances at a time.")
+            abort
+          else
+            puts "Querying server for scheduled benchmark tests..."
+            benchmarkRuns = client.searchAny('benchmarkRuns', 'findByBenchmarkInstanceAgentInstanceIdAndStatus',
+                    { :agentInstanceId => agentInstanceId, :status => 'scheduled' })
+                      
+            benchmarkRuns = benchmarkRuns["content"]; # TODO: better put to rest client helper
+              
+            # if we have benchmarks => run
+            if (benchmarkRuns.any?)
+              puts "Received " + benchmarkRuns.length.to_s + " scheduled tests, starting execution..."
+              benchmarkRuns.each do |benchmarkRun|
+                reset()
+                @benchmarkRun = Benchmark::BenchmarkRun.new(benchmarkRun)
+                executeBenchmark()
+              end
+            else
+              puts Time.now.utc.to_s + ": There are currently no tests scheduled for this agent. Running execution again in two minutes."
+              sleep 120
+            end
+          end
+        end until false
+      end
+      
+      def executeBenchmark()
+        @benchmarkTest = @benchmarkRun.benchmarkTest
+        @environment.push(@settings["env"]["numberOfRunsVar"] + "=" + @benchmarkTest["numberOfRuns"].to_s)
+        @benchmarkRunResult = Benchmark::BenchmarkRunResult.new(@benchmarkRun)
+        
+          
+        puts "Executing BenchmarkTest " + @benchmarkTest["name"];
+        
+        # install benchmark test
+        # update status on server 
+        @benchmarkRun.running!("Installing BenchmarkTest " + @benchmarkTest["name"]);
+        post_status_update()
+        batchInstall()
+        
+        # execute benchmark test
+        # update status on server 
+        @benchmarkRun.running!("Running BenchmarkTest " + @benchmarkTest["name"]);
+        post_status_update()
+        batchRun()
+        
+        # Post results
+        # Replace object with url
+        @benchmarkRunResult.benchmarkRun = RestClientWrapper.load_entity_ref(@benchmarkRunResult.benchmarkRun.to_h)["href"]
+          
+        # post result entity
+        client.post("benchmarkRunResults", @benchmarkRunResult)
+        
+        puts "Results posted to server."
+        
+      end
+      
+      def batchInstall
+        installCommand = compileParams("batch-install", testLabel(@benchmarkTest))
+        status = POpen4.popen4("sh") do | stdout, stderr, stdin, pid |
+            stdin.puts @settings["executable"] + " " + installCommand
+            stdin.close
+            
+            puts "pid        : #{ pid }"  
+            puts "stdout     : #{ stdout.read.strip }"
+            puts "stderr     : #{ stderr.read.strip }"        
+        end
+      end
+      
+      def batchRun
+        installCommand = compileParams("batch-run", testLabel(@benchmarkTest))
+        startStamp = Time.now
+        puts "Start time: "+startStamp.utc.to_s
+        if (!@options["development"]) 
+          status = POpen4.popen4("sh") do | stdout, stderr, stdin, pid |
+              stdin.puts @settings["executable"] + " " + installCommand
+              stdin.close
+              
+              puts "pid        : #{ pid }"  
+              puts "stdout     : #{ stdout.read.strip }"
+              puts "stderr     : #{ stderr.read.strip }"        
+          end
+        end
+        stopStamp = Time.now
+        puts "End time: "+stopStamp.utc.to_s
+        executionTime = (stopStamp - startStamp) # / @nanoSeconds;
+        
+        puts "Benchmark " +  testLabel(@benchmarkTest) + " executed in " + executionTime.to_s + " seconds"    
+  
+        
+        # storing benchmarking execution gross timer
+        resultValueTimer = Benchmark::BenchmarkRunResultValue.new()
+        resultValueTimer.name = "ExecRuntime"
+        resultValueTimer.description = "Benchmark runtime measured by execution handler"
+        resultValueTimer.value = executionTime.to_s
+        resultValueTimer.scale = "Seconds"
+        # add timer value to result
+        @benchmarkRunResult.pushResultValue(resultValueTimer)
+        
+        # get results here
+        puts "Collecting results from xml"
+        # walk result files (usually only one)
+        files = resultFiles()
+        files.each do |f|
+          puts "Processing #{f}..."
+          
+          # read result file contents
+          resultxml = File.read(f)
+          if (resultxml == nil) 
+            @benchmarkRun.failed!("ERROR, no xml results found")
+            post_status_update()
+            return
+          end
+  
+          # convert xml to result objects
+          results = compileResults(resultxml)
+          if (! results.empty?)
+            @benchmarkRun.succeed!("OK, finished successfully.")
+          else
+            @benchmarkRun.failed!("ERROR, no results")
+          end
+          post_status_update()
+  
+        end
+        
+      end # batchRun
+      
+      
+      def compileResults(xml)
+                
+        @benchmarkRunResult.rawBenchmarkOutput = xml
+        @benchmarkRunResult.ip_address = @ip_address
+        @benchmarkRunResult.hostname = @hostname
+                  
+        # parse result xml and convert to object
+        results = XmlSimple.xml_in(xml)
+        
+        # loop over results and create 
+        results['Result'].each do |res|
+          
+          res['Data'].each do |entry|
+            
+            # create BenchmarkRunResultValue - copy values from result XML
+            value = Benchmark::BenchmarkRunResultValue.new()
+            value.name = results['System'][0]['Identifier'][0]
+            value.timestamp = results['System'][0]['Timestamp']
+            value.description = res['Description'][0] || results["Generated"][0]["TestClient"][0] + "\n" + results["Generated"][0]["Description"][0]
+            value.arguments = res['Arguments'][0].empty? ? "" : res['Arguments'][0]
+            value.scale = res['Scale'][0]
+            value.value = entry['Entry'][0]['Value'][0]
+            value.rawString = entry['Entry'][0]['RawString'][0]
+             
+            # add benchmark to result instance var
+            @benchmarkRunResult.pushResultValue(value)
+            
+            value
+            
+          end
+        end
+      
+      end
+      
+      
+      def resultFiles
+        puts "Loading files from " + resultDir + @fileSeparator + "**/test-1.xml"
+        Dir[resultDir() + @fileSeparator + "**/test-1.xml"]  ## TODO CHECK if this yields one file only and the right one, too :)
+      end
+      
+      def reset()
+        @benchmarkRun = nil
+        @benchmarkRunResult = nil
+        if (!@options["development"] && File.exists?(resultDir))
+          FileUtils.rm_rf(resultDir)
+        end
+      end
+      
+      def post_status_update
+        # post benchmarkRun status update
+        client.patch("benchmarkRuns", @benchmarkRun.id, @benchmarkRun.get_status_hash())
+        # TODO: error handling
+      end
+  
+          
+    end
+  end  
+end
+