testbot_instructure 0.7.8

data/lib/requester/requester.rb

@@ -0,0 +1,171 @@
+require 'rubygems'
+require 'httparty'
+require 'ostruct'
+require 'erb'
+require File.dirname(__FILE__) + '/../shared/ssh_tunnel'
+require File.expand_path(File.dirname(__FILE__) + '/../shared/testbot')
+
+class Hash
+  def symbolize_keys_without_active_support
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+    options
+    end
+  end
+end
+
+module Testbot::Requester
+
+  class Requester
+
+    attr_reader :config
+
+    def initialize(config = {})
+      config = config.symbolize_keys_without_active_support
+      config[:rsync_path]             ||= Testbot::DEFAULT_SERVER_PATH
+      config[:project]                ||= Testbot::DEFAULT_PROJECT
+      config[:server_user]            ||= Testbot::DEFAULT_USER
+      config[:available_runner_usage] ||= Testbot::DEFAULT_RUNNER_USAGE
+      @config = OpenStruct.new(config)
+    end
+
+    def run_tests(adapter, dir)
+      puts if config.simple_output || config.logging
+
+      if config.ssh_tunnel
+        log "Setting up ssh tunnel" do
+          SSHTunnel.new(config.server_host, config.server_user, adapter.requester_port).open
+        end
+        server_uri = "http://127.0.0.1:#{adapter.requester_port}"
+      else
+        server_uri = "http://#{config.server_host}:#{Testbot::SERVER_PORT}"
+      end
+
+      log "Syncing files" do
+        rsync_ignores = config.rsync_ignores.to_s.split.map { |pattern| "--exclude='#{pattern}'" }.join(' ')
+        system("rsync -az --delete --delete-excluded -e ssh #{rsync_ignores} . #{rsync_uri}")
+
+        exitstatus = $?.exitstatus
+        unless exitstatus == 0
+          puts "rsync failed with exit code #{exitstatus}"
+          exit 1
+        end
+      end
+
+      files = adapter.test_files(dir)
+      sizes = adapter.get_sizes(files)
+
+      build_id = nil
+      log "Requesting run" do
+        response = HTTParty.post("#{server_uri}/builds", :body => { :root => root,
+                                 :type => adapter.type.to_s,
+                                 :project => config.project,
+                                 :available_runner_usage => config.available_runner_usage,
+                                 :files => files.join(' '),
+                                 :sizes => sizes.join(' '),
+                                 :jruby => jruby? }).response
+
+        if response.code == "503"
+          puts "No runners available. If you just started a runner, try again. It usually takes a few seconds before they're available."
+          return false
+        elsif response.code != "200"
+          puts "Could not create build, #{response.code}: #{response.body}"
+          return false
+        else
+          build_id = response.body
+        end
+      end
+
+      at_exit do
+        unless ENV['IN_TEST'] || @done
+          log "Notifying server we want to stop the run" do
+            HTTParty.delete("#{server_uri}/builds/#{build_id}")
+          end
+        end
+      end
+
+      puts if config.logging
+
+      last_results_size = 0
+      success = true
+      error_count = 0
+      while true
+        sleep 0.5
+
+        begin
+          @build = HTTParty.get("#{server_uri}/builds/#{build_id}", :format => :json)
+          next unless @build
+        rescue Exception => ex
+          error_count += 1
+          if error_count > 4
+            puts "Failed to get status: #{ex.message}"
+            error_count = 0
+          end
+          next
+        end
+
+        results = @build['results'][last_results_size..-1]
+        unless results == ''
+          if config.simple_output
+            print results.gsub(/[^\.F]|Finished/, '')
+            STDOUT.flush
+          else
+            print results
+            STDOUT.flush
+          end
+        end
+
+        last_results_size = @build['results'].size
+
+        break if @build['done']
+      end
+
+      puts if config.simple_output
+
+      if adapter.respond_to?(:sum_results)
+        puts "\n" + adapter.sum_results(@build['results'])
+      end
+
+      @done = true
+      @build["success"]
+    end
+
+    def self.create_by_config(path)
+      Requester.new(YAML.load(ERB.new(File.open(path).read).result))
+    end
+
+    private
+
+    def log(text)
+      if config.logging
+        print "#{text}... "; STDOUT.flush
+        yield
+        puts "done"
+      else
+        yield
+      end
+    end
+
+    def root
+      if localhost?
+        config.rsync_path
+      else
+        "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
+      end
+    end
+
+    def rsync_uri
+      localhost? ? config.rsync_path : "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
+    end
+
+    def localhost?
+      [ '0.0.0.0', 'localhost', '127.0.0.1' ].include?(config.server_host)
+    end
+
+    def jruby?
+      RUBY_PLATFORM =~ /java/ || !!ENV['USE_JRUBY']
+    end
+
+  end
+
+end

data/lib/runner/job.rb

@@ -0,0 +1,112 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'runner.rb'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'safe_result_text.rb'))
+require 'posix/spawn'
+
+module Testbot::Runner
+  class Job
+    attr_reader :root, :project, :build_id
+
+    TIME_TO_WAIT_BETWEEN_POSTING_RESULTS = 5
+
+    def initialize(runner, id, build_id, project, root, type, ruby_interpreter, files)
+      @runner, @id, @build_id, @project, @root, @type, @ruby_interpreter, @files =
+        runner, id, build_id, project, root, type, ruby_interpreter, files
+      @success = true
+    end
+
+    def jruby?
+      @ruby_interpreter == 'jruby'
+    end
+
+    def run(instance)
+      return if @killed
+      puts "Running job #{@id} (build #{@build_id})... "
+      test_env_number = (instance == 0) ? '' : instance + 1
+      result = "\n#{`hostname`.chomp}:#{Dir.pwd}\n"
+      base_environment = "export RAILS_ENV=test; export TEST_ENV_NUMBER=#{test_env_number}; cd #{@project};"
+
+      adapter = Adapter.find(@type)
+      run_time = measure_run_time do
+        result += run_and_return_result("#{base_environment} #{adapter.command(@project, ruby_cmd, @files)}")
+      end
+
+      Server.put("/jobs/#{@id}", :body => { :result => SafeResultText.clean(result), :status => status, :time => run_time })
+      puts "Job #{@id} finished."
+    end
+
+    def kill!(build_id)
+      if @build_id == build_id && @pid
+        kill_processes
+        @killed = true
+      end
+    end
+
+    private
+
+    def kill_processes
+      # Kill process and its children (processes in the same group)
+      Process.kill('KILL', -@pid) rescue :failed_to_kill_process
+    end
+
+    def status
+      success? ? "successful" : "failed"
+    end
+
+    def measure_run_time
+      start_time = Time.now
+      yield
+      (Time.now - start_time) * 100
+    end
+
+    def post_results(output)
+      Server.put("/jobs/#{@id}", :body => { :result => SafeResultText.clean(output), :status => "building" })
+    rescue Timeout::Error
+      puts "Got a timeout when posting an job result update. This can happen when the server is busy and is not a critical error."
+    end
+
+    def run_and_return_result(command)
+      read_pipe = spawn_process(command)
+
+      output = ""
+      last_post_time = Time.now
+      while char = read_pipe.getc
+        char = (char.is_a?(Fixnum) ? char.chr : char) # 1.8 <-> 1.9
+        output << char
+        if Time.now - last_post_time > TIME_TO_WAIT_BETWEEN_POSTING_RESULTS
+          post_results(output)
+          last_post_time = Time.now
+        end
+      end
+
+      # Kill child processes, if any
+      kill_processes
+
+      output
+    end
+
+    def spawn_process(command)
+      read_pipe, write_pipe = IO.pipe
+      @pid = POSIX::Spawn::spawn(command, :err => write_pipe, :out => write_pipe, :pgroup => true)
+
+      Thread.new do
+        Process.waitpid(@pid)
+        @success = ($?.exitstatus == 0)
+        write_pipe.close
+      end
+
+      read_pipe
+    end
+
+    def success?
+      @success
+    end
+
+    def ruby_cmd
+      if @ruby_interpreter == 'jruby' && @runner.config.jruby_opts
+        'jruby ' + @runner.config.jruby_opts
+      else
+        @ruby_interpreter
+      end
+    end
+  end
+end

data/lib/runner/runner.rb

@@ -0,0 +1,222 @@
+require 'rubygems'
+require 'httparty'
+require 'ostruct'
+require File.expand_path(File.dirname(__FILE__) + '/../shared/ssh_tunnel')
+require File.expand_path(File.dirname(__FILE__) + '/../shared/adapters/adapter')
+require File.expand_path(File.dirname(__FILE__) + '/job')
+
+module Testbot::Runner
+  TIME_BETWEEN_NORMAL_POLLS = 1
+  TIME_BETWEEN_QUICK_POLLS = 0.1
+  TIME_BETWEEN_PINGS = 5
+  TIME_BETWEEN_VERSION_CHECKS = Testbot.version.include?('.DEV.') ? 10 : 60
+
+  class CPU
+
+    def self.count
+      case RUBY_PLATFORM
+      when /darwin/
+        `sysctl machdep.cpu.core_count | awk '{ print $2 }'`.to_i
+      when /linux/
+        `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+      end
+    end
+
+  end
+
+  class Server
+    include HTTParty
+    default_timeout 10
+  end
+
+  class Runner
+
+    def initialize(config)
+      @instances = []
+      @last_build_id = nil
+      @last_version_check = Time.now - TIME_BETWEEN_VERSION_CHECKS - 1
+      @config = OpenStruct.new(config)
+      @config.max_instances = @config.max_instances ? @config.max_instances.to_i : CPU.count 
+
+      if @config.ssh_tunnel
+        server_uri = "http://127.0.0.1:#{Testbot::SERVER_PORT}"
+      else
+        server_uri = "http://#{@config.server_host}:#{Testbot::SERVER_PORT}"
+      end
+
+      Server.base_uri(server_uri)
+    end
+
+    attr_reader :config
+
+    def run!
+      # Remove legacy instance* and *_rsync|git style folders
+      Dir.entries(".").find_all { |name| name.include?('instance') || name.include?('_rsync') ||
+        name.include?('_git') }.each { |folder|
+        system "rm -rf #{folder}"
+      }
+
+      SSHTunnel.new(@config.server_host, @config.server_user || Testbot::DEFAULT_USER).open if @config.ssh_tunnel
+      while true
+        begin
+          update_uid!
+          start_ping
+          wait_for_jobs
+        rescue Exception => ex
+          break if [ 'SignalException', 'Interrupt' ].include?(ex.class.to_s)
+          puts "The runner crashed, restarting. Error: #{ex.inspect} #{ex.class}"
+        end
+      end
+    end
+
+    private
+
+    def update_uid!
+      # When a runner crashes or is restarted it might loose current job info. Because
+      # of this we provide a new unique ID to the server so that it does not wait for
+      # lost jobs to complete.
+      @uid = "#{Time.now.to_i * rand}"
+    end
+
+    def wait_for_jobs
+      last_check_found_a_job = false
+      loop do
+        sleep (last_check_found_a_job ? TIME_BETWEEN_QUICK_POLLS : TIME_BETWEEN_NORMAL_POLLS)
+
+        check_for_update if !last_check_found_a_job && time_for_update?
+
+        # Only get jobs from one build at a time
+        next_params = base_params
+        if @instances.size > 0
+          next_params.merge!({ :build_id => @last_build_id })
+          next_params.merge!({ :no_jruby => true }) if max_jruby_instances?
+        else
+          @last_build_id = nil
+        end
+
+        # Makes sure all instances are listed as available after a run
+        clear_completed_instances 
+
+        next_job = Server.get("/jobs/next", :query => next_params) rescue nil
+        last_check_found_a_job = (next_job != nil && next_job.body != "")
+        next unless last_check_found_a_job
+
+        job = Job.new(*([ self, next_job.split(',') ].flatten))
+        if first_job_from_build?
+          fetch_code(job)
+          before_run(job)
+        end
+
+        @last_build_id = job.build_id
+
+        # Must be outside the thread or it will sometimes run
+        # multiple jobs using the same instance number.
+        instance_number = free_instance_number
+
+        @instances << [ Thread.new { job.run(instance_number) }, instance_number, job ]
+
+        loop do
+          clear_completed_instances
+          break unless max_instances_running?
+        end
+      end
+    end
+
+    def max_jruby_instances?
+      return unless @config.max_jruby_instances
+      @instances.find_all { |thread, n, job| job.jruby? }.size >= @config.max_jruby_instances
+    end
+
+    def fetch_code(job)
+      system "rsync -az --delete --delete-excluded -e ssh #{job.root}/ #{job.project}"
+    end
+
+    def before_run(job)
+      rvm_prefix = RubyEnv.rvm_prefix(job.project)
+      bundler_cmd = (RubyEnv.bundler?(job.project) ? [rvm_prefix, "bundle &&", rvm_prefix, "bundle exec"] : [rvm_prefix]).compact.join(" ")
+      command_prefix = "cd #{job.project} && export RAILS_ENV=test && export TEST_INSTANCES=#{@config.max_instances} && #{bundler_cmd}"
+
+      if File.exists?("#{job.project}/lib/tasks/testbot.rake")
+        system "#{command_prefix} rake testbot:before_run"
+      elsif File.exists?("#{job.project}/config/testbot/before_run.rb")
+        system "#{command_prefix} ruby config/testbot/before_run.rb"
+      else
+        # workaround to bundle within the correct env
+        system "#{command_prefix} ruby -e ''"
+      end
+    end
+
+    def first_job_from_build?
+      @last_build_id == nil
+    end
+
+    def time_for_update?
+      time_for_update = ((Time.now - @last_version_check) >= TIME_BETWEEN_VERSION_CHECKS)
+      @last_version_check = Time.now if time_for_update
+      time_for_update
+    end
+
+    def check_for_update
+      return unless @config.auto_update
+      version = Server.get('/version') rescue Testbot.version
+      return unless version != Testbot.version
+
+      # In a PXE cluster with a shared gem folder we only want one of them to do the update
+      if @config.wait_for_updated_gem
+        # Gem.available? is cached so it won't detect new gems.
+        gem = Gem::Dependency.new("testbot", version)
+        successful_install = !Gem::SourceIndex.from_installed_gems.search(gem).empty?
+      else
+        if version.include?(".DEV.")
+          successful_install = system("wget #{@config.dev_gem_root}/testbot-#{version}.gem && gem install testbot-#{version}.gem --no-ri --no-rdoc && rm testbot-#{version}.gem")
+        else
+          successful_install = system "gem install testbot -v #{version} --no-ri --no-rdoc"
+        end
+      end
+
+      system "testbot #{ARGV.join(' ')}" if successful_install
+    end
+
+    def ping_params
+      { :hostname => (@hostname ||= `hostname`.chomp), :max_instances => @config.max_instances,
+        :idle_instances => (@config.max_instances - @instances.size), :username => ENV['USER'], :build_id => @last_build_id }.merge(base_params)
+    end
+
+    def base_params
+      { :version => Testbot.version, :uid => @uid }
+    end
+
+    def max_instances_running?
+      @instances.size == @config.max_instances
+    end
+
+    def clear_completed_instances
+      @instances.each_with_index do |data, index|
+        @instances.delete_at(index) if data.first.join(0.25)
+      end
+    end
+
+    def free_instance_number
+      0.upto(@config.max_instances - 1) do |number|
+        return number unless @instances.find { |instance, n, job| n == number }
+      end
+    end
+
+    def start_ping
+      Thread.new do
+        while true
+          begin
+            response = Server.get("/runners/ping", :body => ping_params).body
+            if response.include?('stop_build')
+              build_id = response.split(',').last
+              @instances.each { |instance, n, job| job.kill!(build_id) }
+            end
+          rescue
+          end
+          sleep TIME_BETWEEN_PINGS
+        end
+      end
+    end
+
+  end
+end

data/lib/runner/safe_result_text.rb

@@ -0,0 +1,29 @@
+require 'iconv'
+
+module Testbot::Runner
+  class SafeResultText
+    def self.clean(text)
+      clean_escape_sequences(strip_invalid_utf8(text))
+    end
+
+    def self.strip_invalid_utf8(text)
+      # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+      ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      ic.iconv(text + ' ')[0..-2]
+    end
+
+    def self.clean_escape_sequences(text)
+      tail_marker = "^[[0m"
+      tail = text.rindex(tail_marker) && text[text.rindex(tail_marker)+tail_marker.length..-1]
+      if !tail
+        text
+      elsif tail.include?("^[[") && !tail.include?("m")
+        text[0..text.rindex(tail_marker) + tail_marker.length - 1]
+      elsif text.scan(/\[.*?m/).last != tail_marker
+        text[0..text.rindex(tail_marker) + tail_marker.length - 1]
+      else
+        text
+      end
+    end
+  end
+end

data/lib/server/build.rb

@@ -0,0 +1,36 @@
+module Testbot::Server
+
+  class Build < MemoryModel
+
+    def initialize(hash)
+      super({ :success => true, :done => false, :results => '' }.merge(hash))
+    end
+
+    def self.create_and_build_jobs(hash)
+      hash["jruby"] = (hash["jruby"] == "true") ? 1 : 0
+      build = create(hash.reject { |k, v| k == 'available_runner_usage' })
+      build.create_jobs!(hash['available_runner_usage'])
+      build
+    end
+
+    def create_jobs!(available_runner_usage)
+      groups = Group.build(self.files.split, self.sizes.split.map { |size| size.to_i },
+                           Runner.total_instances.to_f * (available_runner_usage.to_i / 100.0), self.type)
+      groups.each do |group|
+        Job.create(:files => group.join(' '),
+                   :root => self.root,
+                   :project => self.project,
+                   :type => self.type,
+                   :build => self,
+                   :jruby => self.jruby)
+      end
+    end
+
+    def destroy
+      Job.all.find_all { |j| j.build == self }.each { |job| job.destroy }
+      super
+    end
+
+  end
+
+end

data/lib/server/group.rb

@@ -0,0 +1,48 @@
+require 'rubygems'
+
+module Testbot::Server
+
+  class Group
+
+    DEFAULT = nil
+
+    def self.build(files, sizes, instance_count, type)
+      tests_with_sizes = slow_tests_first(map_files_and_sizes(files, sizes))
+
+      groups = []
+      current_group, current_size = 0, 0
+      tests_with_sizes.each do |test, size|
+        # inserts into next group if current is full and we are not in the last group
+        if (0.5*size + current_size) > group_size(tests_with_sizes, instance_count) and instance_count > current_group + 1
+          current_size = size
+          current_group += 1
+        else
+          current_size += size
+        end
+        groups[current_group] ||= []
+        groups[current_group] << test
+      end
+
+      groups.compact
+    end
+
+    private
+
+    def self.group_size(tests_with_sizes, group_count)
+      total = tests_with_sizes.inject(0) { |sum, test| sum += test[1] }
+      total / group_count.to_f
+    end
+
+    def self.map_files_and_sizes(files, sizes)
+      list = []
+      files.each_with_index { |file, i| list << [ file, sizes[i] ] }
+      list
+    end
+
+    def self.slow_tests_first(tests)
+      tests.sort_by { |test, time| time.to_i }.reverse
+    end
+
+  end
+
+end

data/lib/server/job.rb

@@ -0,0 +1,64 @@
+module Testbot::Server
+
+  class Job < MemoryModel
+
+    def update(hash)
+      super(hash)
+      if self.build
+        self.done = done?
+        done = !Job.all.find { |j| !j.done && j.build == self.build }
+        self.build.update(:results => build_results(build), :done => done)
+
+        build_broken_by_job = (self.status == "failed" && build.success)
+        self.build.update(:success => false) if build_broken_by_job
+      end
+    end
+
+    def self.next(params, remove_addr)
+      clean_params = params.reject { |k, v| k == "no_jruby" }
+      runner = Runner.record! clean_params.merge({ :ip => remove_addr, :last_seen_at => Time.now })
+      return unless Server.valid_version?(params[:version])
+      [ next_job(params["build_id"], params["no_jruby"]), runner ]
+    end
+
+    private
+
+    def build_results(build)
+      self.last_result_position ||= 0
+      new_results = self.result.to_s[self.last_result_position..-1] || ""
+      self.last_result_position = self.result.to_s.size
+
+      # Don't know why this is needed as the job should cleanup
+      # escape sequences.
+      if new_results[0,4] == '[32m'
+        new_results = new_results[4..-1]
+      end
+
+      build.results.to_s + new_results 
+    end
+
+    def done?
+      self.status == "successful" || self.status == "failed"
+    end
+
+    def self.next_job(build_id, no_jruby)
+      release_jobs_taken_by_missing_runners!
+      jobs = Job.all.find_all { |j|
+        !j.taken_at &&
+          (build_id ? j.build.id.to_s == build_id : true) &&
+          (no_jruby ? j.jruby != 1 : true)
+      }
+
+      jobs[rand(jobs.size)]
+    end
+
+    def self.release_jobs_taken_by_missing_runners!
+      missing_runners = Runner.all.find_all { |r| r.last_seen_at < (Time.now - Runner.timeout) }
+      missing_runners.each { |runner|
+        Job.all.find_all { |job| job.taken_by == runner }.each { |job| job.update(:taken_at => nil) }
+      }
+    end
+
+  end
+
+end