testbot 0.4.6 → 0.4.7

data/lib/requester.rb

@@ -1,130 +0,0 @@
-require 'rubygems'
-require 'httparty'
-require 'macaddr'
-require 'ostruct'
-require File.dirname(__FILE__) + '/shared/ssh_tunnel'
-require File.dirname(__FILE__) + '/adapters/adapter'
-require File.expand_path(File.dirname(__FILE__) + '/testbot')
-
-class Hash
-  def symbolize_keys_without_active_support
-    inject({}) do |options, (key, value)|
-      options[(key.to_sym rescue key) || key] = value
-      options
-    end
-  end
-end
-
-class Requester
-
-  attr_reader :config
-
-  def initialize(config = {})
-    config = config.symbolize_keys_without_active_support
-    config[:rsync_path]             ||= Testbot::DEFAULT_SERVER_PATH
-    config[:project]                ||= Testbot::DEFAULT_PROJECT
-    config[:server_user]            ||= Testbot::DEFAULT_USER
-    config[:available_runner_usage] ||= Testbot::DEFAULT_RUNNER_USAGE
-    @config = OpenStruct.new(config)
-  end
-
-  def run_tests(adapter, dir)
-    puts if config.simple_output
-
-    if config.ssh_tunnel
-      SSHTunnel.new(config.server_host, config.server_user, adapter.requester_port).open
-      server_uri = "http://127.0.0.1:#{adapter.requester_port}"
-    else
-      server_uri = "http://#{config.server_host}:#{Testbot::SERVER_PORT}"
-    end
-
-    rsync_ignores = config.rsync_ignores.to_s.split.map { |pattern| "--exclude='#{pattern}'" }.join(' ')
-    system "rsync -az --delete -e ssh #{rsync_ignores} . #{rsync_uri}"
-   
-    files = adapter.test_files(dir) 
-    sizes = adapter.get_sizes(files)
-
-    build_id = HTTParty.post("#{server_uri}/builds", :body => { :root => root,
-                                                     :type => adapter.type.to_s,
-                                                     :project => config.project,
-                                                     :requester_mac => Mac.addr,
-                                                     :available_runner_usage => config.available_runner_usage,
-                                                     :files => files.join(' '),
-                                                     :sizes => sizes.join(' '),
-                                                     :jruby => jruby? })
-
-
-    last_results_size = 0
-    success = true
-    error_count = 0
-    while true
-      sleep 1
-
-      begin
-        @build = HTTParty.get("#{server_uri}/builds/#{build_id}", :format => :json)
-        next unless @build
-      rescue Exception => ex
-        error_count += 1
-        if error_count > 4
-          puts "Failed to get status: #{ex.message}"
-          error_count = 0
-        end
-        next
-      end
-
-      results = @build['results'][last_results_size..-1]
-      unless results == ''
-        if config.simple_output
-          print results.gsub(/[^\.F]|Finished/, '')
-          STDOUT.flush
-        else
-          puts results
-        end
-      end
-
-      last_results_size = @build['results'].size
-
-      break if @build['done']
-    end
-
-    puts if config.simple_output
-
-    @build["success"]
-  end
-
-  def self.create_by_config(path)
-    config = YAML.load_file(path)
-    Requester.new(config)
-  end
-
-  def result_lines
-    @build['results'].split("\n").find_all { |line| line_is_result?(line) }.map { |line| line.chomp }
-  end
-
-  private
-
-  def root
-    if localhost?
-      config.rsync_path
-    else
-      "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
-    end
-  end
-
-  def rsync_uri
-    localhost? ? config.rsync_path : "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
-  end
-
-  def localhost?
-    [ '0.0.0.0', 'localhost', '127.0.0.1' ].include?(config.server_host)
-  end
-
-  def line_is_result?(line)
-    line =~ /\d+ fail/
-  end
-
-  def jruby?
-    RUBY_PLATFORM =~ /java/ || !!ENV['USE_JRUBY']
-  end
-
-end

data/lib/runner.rb

@@ -1,213 +0,0 @@
-require 'rubygems'
-require 'httparty'
-require 'macaddr'
-require 'ostruct'
-require File.dirname(__FILE__) + '/shared/ssh_tunnel'
-require File.dirname(__FILE__) + '/adapters/adapter'
-require File.dirname(__FILE__) + '/runner/job'
-
-TIME_BETWEEN_NORMAL_POLLS = 1
-TIME_BETWEEN_QUICK_POLLS = 0.1
-TIME_BETWEEN_PINGS = 5
-TIME_BETWEEN_VERSION_CHECKS = Testbot.version.include?('.DEV.') ? 10 : 60
-MAX_CPU_USAGE_WHEN_IDLE = 50
-
-class CPU
-
- def self.current_usage
-   process_usages = `ps -eo pcpu`
-   total_usage = process_usages.split("\n").inject(0) { |sum, usage| sum += usage.strip.to_f }
-   (total_usage / count).to_i
- end
-
- def self.count
-   case RUBY_PLATFORM
-     when /darwin/
-       `hwprefs cpu_count`.to_i
-     when /linux/
-       `cat /proc/cpuinfo | grep processor | wc -l`.to_i
-   end
- end
-
-end
-
-class Server
-  include HTTParty
-end
-
-class Runner
-
-  def initialize(config)
-    @instances = []
-    @last_requester_mac = nil
-    @last_version_check = Time.now - TIME_BETWEEN_VERSION_CHECKS - 1
-    @config = OpenStruct.new(config)
-    @config.max_instances = @config.max_instances ? @config.max_instances.to_i : CPU.count 
-    
-    if @config.ssh_tunnel
-      server_uri = "http://127.0.0.1:#{Testbot::SERVER_PORT}"
-    else
-      server_uri = "http://#{@config.server_host}:#{Testbot::SERVER_PORT}"
-    end
-    
-    Server.base_uri(server_uri)
-  end
-  
-  attr_reader :config
-  
-  def run!
-    # Remove legacy instance* and *_rsync|git style folders
-    Dir.entries(".").find_all { |name| name.include?('instance') || name.include?('_rsync') ||
-                                       name.include?('_git') }.each { |folder|
-      system "rm -rf #{folder}"
-    }
-    
-    SSHTunnel.new(@config.server_host, @config.server_user || Testbot::DEFAULT_USER).open if @config.ssh_tunnel
-    while true
-      begin
-        update_uid!
-        start_ping
-        wait_for_jobs
-      rescue Exception => ex
-        break if [ 'SignalException', 'Interrupt' ].include?(ex.class.to_s)
-        puts "The runner crashed, restarting. Error: #{ex.inspect} #{ex.class}"
-      end
-    end
-  end
-
-  private
-  
-  def update_uid!
-    # When a runner crashes or is restarted it might loose current job info. Because
-    # of this we provide a new unique ID to the server so that it does not wait for
-    # lost jobs to complete.
-    @uid = "#{Time.now.to_i}@#{Mac.addr}"
-  end
-  
-  def wait_for_jobs
-    last_check_found_a_job = false
-    loop do
-      sleep (last_check_found_a_job ? TIME_BETWEEN_QUICK_POLLS : TIME_BETWEEN_NORMAL_POLLS)
-
-      check_for_update if !last_check_found_a_job && time_for_update?
-
-      # Only get jobs from one requester at a time
-      next_params = base_params
-      if @instances.size > 0
-        next_params.merge!({ :requester_mac => @last_requester_mac })
-        next_params.merge!({ :no_jruby => true }) if max_jruby_instances?
-      else
-        @last_requester_mac = nil
-      end
-      
-      # Makes sure all instances are listed as available after a run
-      clear_completed_instances 
-      next unless cpu_available?
-      
-      next_job = Server.get("/jobs/next", :query => next_params) rescue nil
-      last_check_found_a_job = (next_job != nil)
-      next unless last_check_found_a_job
-
-      job = Job.new(*([ self, next_job.split(',') ].flatten))
-      if first_job_from_requester?
-        fetch_code(job)
-        before_run(job) if File.exists?("#{job.project}/lib/tasks/testbot.rake")
-      end
-      
-      @instances << [ Thread.new { job.run(free_instance_number) },
-                      free_instance_number, job ]
-      @last_requester_mac = job.requester_mac
-      loop do
-        clear_completed_instances
-        break unless max_instances_running?
-      end
-    end
-  end
-  
-  def max_jruby_instances?
-    return unless @config.max_jruby_instances
-    @instances.find_all { |thread, n, job| job.jruby? }.size >= @config.max_jruby_instances
-  end
-  
-  def fetch_code(job)
-    system "rsync -az --delete -e ssh #{job.root}/ #{job.project}"
-  end
-  
-  def before_run(job)
-    bundler_cmd = RubyEnv.bundler?(job.project) ? "bundle; " : ""
-    system "export RAILS_ENV=test; export TEST_INSTANCES=#{@config.max_instances}; cd #{job.project}; #{bundler_cmd} rake testbot:before_run"
-  end
-  
-  def first_job_from_requester?
-    @last_requester_mac == nil
-  end
-  
-  def cpu_available?
-    @instances.size > 0 || CPU.current_usage < MAX_CPU_USAGE_WHEN_IDLE
-  end
-  
-  def time_for_update?
-    time_for_update = ((Time.now - @last_version_check) >= TIME_BETWEEN_VERSION_CHECKS)
-    @last_version_check = Time.now if time_for_update
-    time_for_update
-  end
-  
-  def check_for_update
-    return unless @config.auto_update
-    version = Server.get('/version') rescue Testbot.version
-    return unless version != Testbot.version
-
-    # In a PXE cluster with a shared gem folder we only want one of them to do the update
-    if @config.wait_for_updated_gem
-      # Gem.available? is cached so it won't detect new gems.
-      gem = Gem::Dependency.new("testbot", version)
-      successful_install = !Gem::SourceIndex.from_installed_gems.search(gem).empty?
-    else
-      if version.include?(".DEV.")
-        successful_install = system("wget #{@config.dev_gem_root}/testbot-#{version}.gem && gem install testbot-#{version}.gem --no-ri --no-rdoc && rm testbot-#{version}.gem")
-      else
-        successful_install = system "gem install testbot -v #{version} --no-ri --no-rdoc"
-      end
-    end
-
-    system "testbot #{ARGV.join(' ')}" if successful_install
-  end
-
-  def ping_params
-    { :hostname => (@hostname ||= `hostname`.chomp), :max_instances => @config.max_instances,
-      :idle_instances => (@config.max_instances - @instances.size), :username => ENV['USER'] }.merge(base_params)
-  end
-  
-  def base_params
-    { :version => Testbot.version, :uid => @uid }
-  end
-  
-  def max_instances_running?
-    @instances.size == @config.max_instances
-  end
-
-  def clear_completed_instances
-    @instances.each_with_index do |data, index|
-      @instances.delete_at(index) if data.first.join(0.25)
-    end
-  end
-
-  def free_instance_number
-    0.upto(@config.max_instances - 1) do |number|
-      return number unless @instances.find { |instance, n, job| n == number }
-    end
-  end
-   
-  def start_ping
-    Thread.new do
-      while true
-        begin
-          Server.get("/runners/ping", :body => ping_params)
-        rescue
-        end
-        sleep TIME_BETWEEN_PINGS
-      end
-    end
-  end
-   
-end

data/lib/server.rb

@@ -1,72 +0,0 @@
-require 'rubygems'
-require 'sinatra'
-require 'yaml'
-require 'json'
-require File.join(File.dirname(__FILE__), 'server/job.rb')
-require File.join(File.dirname(__FILE__), 'server/group.rb') unless defined?(Group)
-require File.join(File.dirname(__FILE__), 'server/runner.rb')
-require File.join(File.dirname(__FILE__), 'server/build.rb')
-require File.expand_path(File.join(File.dirname(__FILE__), 'testbot'))
-
-if ENV['INTEGRATION_TEST']
-  set :port, 22880
-else
-  set :port, Testbot::SERVER_PORT
-end
-
-disable :logging if ENV['DISABLE_LOGGING']
-
-class Server
-  def self.valid_version?(runner_version)
-    Testbot.version == runner_version
-  end
-end
-
-post '/builds' do
-  build = Build.create_and_build_jobs(params)[:id].to_s
-end
-
-get '/builds/:id' do
-  build = Build.find(:id => params[:id].to_i)
-  build.destroy if build[:done]
-  { "done" => build[:done], "results" => build[:results], "success" => build[:success] }.to_json
-end
-
-get '/jobs/next' do
-  next_job, runner = Job.next(params, @env['REMOTE_ADDR'])
-  if next_job
-    next_job.update(:taken_at => Time.now, :taken_by_id => runner.id)
-    [ next_job[:id], next_job[:requester_mac], next_job[:project], next_job[:root], next_job[:type], (next_job[:jruby] == 1 ? 'jruby' : 'ruby'), next_job[:files] ].join(',')
-  end
-end
-
-put '/jobs/:id' do
-  Job.find(:id => params[:id].to_i).update(:result => params[:result], :success => params[:success]); nil
-end
-
-get '/runners/ping' do
-  return unless Server.valid_version?(params[:version])
-  runner = Runner.find(:uid => params[:uid])
-  runner.update(params.merge({ :last_seen_at => Time.now })) if runner
-  nil
-end
-
-get '/runners/outdated' do
-  Runner.find_all_outdated.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid] ].join(' ') }.join("\n").strip
-end
-
-get '/runners/available_instances' do
-  Runner.available_instances.to_s
-end
-
-get '/runners/total_instances' do
-  Runner.total_instances.to_s
-end
-
-get '/runners/available' do
-  Runner.find_all_available.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid], runner[:username], runner[:idle_instances] ].join(' ') }.join("\n").strip
-end
-
-get '/version' do
-  Testbot.version
-end