testbot 0.2.6

data/lib/generators/testbot/testbot_generator.rb

@@ -0,0 +1,15 @@
+class TestbotGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+  
+  class_option :connect, :type => :string, :required => true, :desc => "Which server to use (required)"
+  class_option :project, :type => :string, :default => nil, :desc => "The name of your project (default: #{Testbot::DEFAULT_PROJECT})"
+  class_option :rsync_path, :type => :string, :default => nil, :desc => "Sync path on the server (default: #{Testbot::DEFAULT_SERVER_PATH})"  
+  class_option :rsync_ignores, :type => :string, :default => nil, :desc => "Files to rsync_ignores when syncing (default: include all)"
+  class_option :ssh_tunnel, :type => :boolean, :default => nil, :desc => "Use a ssh tunnel"
+  class_option :user, :type => :string, :default => nil, :desc => "Use a custom rsync / ssh tunnel user (default: #{Testbot::DEFAULT_USER})"
+
+  def generate_config
+    template "testbot.yml.erb", "config/testbot.yml"
+    template "testbot.rake.erb", "lib/tasks/testbot.rake"
+  end
+end

data/lib/new_runner.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'httparty'
+
+class Server
+  include HTTParty
+end
+
+class NewRunner
+  
+  def self.run_jobs
+    next_job = Server.get('/jobs/next') rescue nil
+    return unless next_job
+    id, root, specs = next_job.split(',')
+    result = run_and_return_results("export RAILS_ENV=test; export RSPEC_COLOR=true; rake testbot:before_run; script/spec -O spec/spec.opts #{specs}")
+    Server.put("/jobs/#{id}", :body => { :result => result })
+  end
+  
+  def self.load_config
+    host = YAML.load_file("#{ENV['HOME']}/.testbot_runner.yml")[:server_host]
+    Server.base_uri "http://#{host}:2288"
+  end
+
+  private
+
+  # I'd really like to know how to do this better and more testable
+  def self.run_and_return_results(cmd)
+    `#{cmd} 2>&1`
+  end
+  
+end

data/lib/railtie.rb

@@ -0,0 +1,10 @@
+require 'testbot'
+require 'rails'
+
+module Testbot
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path(File.join(File.dirname(__FILE__), "tasks/testbot.rake"))
+    end
+  end
+end

data/lib/requester.rb

@@ -0,0 +1,147 @@
+require 'rubygems'
+require 'httparty'
+require 'macaddr'
+require 'ostruct'
+require File.dirname(__FILE__) + '/shared/ssh_tunnel'
+require File.dirname(__FILE__) + '/adapters/adapter'
+require File.expand_path(File.dirname(__FILE__) + '/testbot')
+
+class Hash
+  def symbolize_keys_without_active_support
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+      options
+    end
+  end
+end
+
+class Requester
+
+  attr_reader :config
+
+  def initialize(config = {})
+    config = config.symbolize_keys_without_active_support
+    config[:rsync_path]             ||= Testbot::DEFAULT_SERVER_PATH
+    config[:project]                ||= Testbot::DEFAULT_PROJECT
+    config[:server_user]            ||= Testbot::DEFAULT_USER
+    config[:available_runner_usage] ||= Testbot::DEFAULT_RUNNER_USAGE
+    @config = OpenStruct.new(config)
+  end
+
+  def run_tests(adapter, dir)
+    puts if config.simple_output
+
+    if config.ssh_tunnel
+      SSHTunnel.new(config.server_host, config.server_user, adapter.requester_port).open
+      server_uri = "http://127.0.0.1:#{adapter.requester_port}"
+    else
+      server_uri = "http://#{config.server_host}:#{Testbot::SERVER_PORT}"
+    end
+
+    rsync_ignores = config.rsync_ignores.to_s.split.map { |pattern| "--exclude='#{pattern}'" }.join(' ')
+    system "rsync -az --delete -e ssh #{rsync_ignores} . #{rsync_uri}"
+   
+    files = find_tests(adapter, dir)
+    sizes = find_sizes(files)
+
+    build_id = HTTParty.post("#{server_uri}/builds", :body => { :root => root,
+                                                     :type => adapter.type.to_s,
+                                                     :project => config.project,
+                                                     :requester_mac => Mac.addr,
+                                                     :available_runner_usage => config.available_runner_usage,
+                                                     :files => files.join(' '),
+                                                     :sizes => sizes.join(' '),
+                                                     :jruby => jruby? })
+
+
+    last_results_size = 0
+    success = true
+    error_count = 0
+    while true
+      sleep 1
+
+      begin
+        @build = HTTParty.get("#{server_uri}/builds/#{build_id}", :format => :json)
+        next unless @build
+      rescue Exception => ex
+        error_count += 1
+        if error_count > 4
+          puts "Failed to get status: #{ex.message}"
+          error_count = 0
+        end
+        next
+      end
+
+      results = @build['results'][last_results_size..-1]
+      unless results == ''
+        if config.simple_output
+          print results.gsub(/[^\.F]|Finished/, '')
+          STDOUT.flush
+        else
+          puts results
+        end
+      end
+
+      last_results_size = @build['results'].size
+
+      success = false if failed_build?(@build)
+      break if @build['done']
+    end
+
+    puts if config.simple_output
+
+    success
+  end
+
+  def self.create_by_config(path)
+    config = YAML.load_file(path)
+    Requester.new(config)
+  end
+
+  def result_lines
+    @build['results'].split("\n").find_all { |line| line_is_result?(line) }.map { |line| line.chomp }
+  end
+
+  private
+
+  def root
+    if localhost?
+      config.rsync_path
+    else
+      "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
+    end
+  end
+
+  def rsync_uri
+    localhost? ? config.rsync_path : "#{config.server_user}@#{config.server_host}:#{config.rsync_path}"
+  end
+
+  def localhost?
+    [ '0.0.0.0', 'localhost', '127.0.0.1' ].include?(config.server_host)
+  end
+
+  def failed_build?(build)
+    result_lines.any? { |line| line_is_failure?(line) }
+  end
+
+  def line_is_result?(line)
+    line =~ /\d+ fail/
+  end
+
+  def line_is_failure?(line)
+    line =~ /(\d{2,}|[1-9]) (fail|error)/
+  end
+
+  def find_tests(adapter, dir)
+    Dir["#{dir}/#{adapter.file_pattern}"]
+  end
+
+  def find_sizes(files)
+    files.map { |file| File.stat(file).size }
+  end
+
+  def jruby?
+    RUBY_PLATFORM =~ /java/ || !!ENV['USE_JRUBY']
+  end
+
+end

data/lib/runner.rb

@@ -0,0 +1,233 @@
+require 'rubygems'
+require 'httparty'
+require 'macaddr'
+require 'ostruct'
+require File.dirname(__FILE__) + '/shared/ssh_tunnel'
+require File.dirname(__FILE__) + '/adapters/adapter'
+
+TIME_BETWEEN_POLLS = 1
+TIME_BETWEEN_PINGS = 5
+TIME_BETWEEN_VERSION_CHECKS = 60
+MAX_CPU_USAGE_WHEN_IDLE = 50
+
+class CPU
+
+ def self.current_usage
+   process_usages = `ps -eo pcpu`
+   total_usage = process_usages.split("\n").inject(0) { |sum, usage| sum += usage.strip.to_f }
+   (total_usage / count).to_i
+ end
+
+ def self.count
+   case RUBY_PLATFORM
+     when /darwin/
+       `hwprefs cpu_count`.to_i
+     when /linux/
+       `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+   end
+ end
+
+end
+
+class Job
+  attr_reader :root, :project, :requester_mac
+    
+  def initialize(runner, id, requester_mac, project, root, type, ruby_interpreter, files)
+    @runner, @id, @requester_mac, @project, @root, @type, @ruby_interpreter, @files =
+         runner, id, requester_mac, project, root, type, ruby_interpreter, files
+  end
+  
+  def jruby?
+    @ruby_interpreter == 'jruby'
+  end
+  
+  def run(instance)
+    puts "Running job #{@id} from #{@requester_mac}... "
+    test_env_number = (instance == 0) ? '' : instance + 1
+    result = "\n#{`hostname`.chomp}:#{Dir.pwd}\n"
+    base_environment = "export RAILS_ENV=test; export TEST_ENV_NUMBER=#{test_env_number}; cd #{@project};"
+    
+    adapter = Adapter.find(@type)
+    result += `#{base_environment} #{adapter.command(ruby_cmd, @files)} 2>&1`
+
+    Server.put("/jobs/#{@id}", :body => { :result => result })
+    puts "Job #{@id} finished."
+  end
+  
+  private
+  
+  def ruby_cmd
+    if @ruby_interpreter == 'jruby' && @runner.config.jruby_opts
+      'jruby ' + @runner.config.jruby_opts
+    else
+      @ruby_interpreter
+    end
+  end
+end
+
+class Server
+  include HTTParty
+end
+
+class Runner
+
+  def initialize(config)
+    @instances = []
+    @last_requester_mac = nil
+    @last_version_check = Time.now - TIME_BETWEEN_VERSION_CHECKS - 1
+    @config = OpenStruct.new(config)
+    @config.max_instances = @config.max_instances ? @config.max_instances.to_i : CPU.count 
+    
+    if @config.ssh_tunnel
+      server_uri = "http://127.0.0.1:#{Testbot::SERVER_PORT}"
+    else
+      server_uri = "http://#{@config.server_host}:#{Testbot::SERVER_PORT}"
+    end
+    
+    Server.base_uri(server_uri)
+  end
+  
+  attr_reader :config
+  
+  def run!
+    # Remove legacy instance* and *_rsync|git style folders
+    Dir.entries(".").find_all { |name| name.include?('instance') || name.include?('_rsync') ||
+                                       name.include?('_git') }.each { |folder|
+      system "rm -rf #{folder}"
+    }
+    
+    SSHTunnel.new(@config.server_host, @config.server_user || Testbot::DEFAULT_USER).open if @config.ssh_tunnel
+    while true
+      begin
+        update_uid!
+        start_ping
+        wait_for_jobs
+      rescue Exception => ex
+        break if [ 'SignalException', 'Interrupt' ].include?(ex.class.to_s)
+        puts "The runner crashed, restarting. Error: #{ex.inspect} #{ex.class}"
+      end
+    end
+  end
+
+  private
+  
+  def update_uid!
+    # When a runner crashes or is restarted it might loose current job info. Because
+    # of this we provide a new unique ID to the server so that it does not wait for
+    # lost jobs to complete.
+    @uid = "#{Time.now.to_i}@#{Mac.addr}"
+  end
+  
+  def wait_for_jobs
+    loop do
+      sleep TIME_BETWEEN_POLLS
+      check_for_update if time_for_update?
+
+      # Only get jobs from one requester at a time
+      next_params = base_params
+      if @instances.size > 0
+        next_params.merge!({ :requester_mac => @last_requester_mac })
+        next_params.merge!({ :no_jruby => true }) if max_jruby_instances?
+      else
+        @last_requester_mac = nil
+      end
+      
+      # Makes sure all instances are listed as available after a run
+      clear_completed_instances 
+      next unless cpu_available?
+      
+      next_job = Server.get("/jobs/next", :query => next_params) rescue nil
+      next if next_job == nil
+      
+      job = Job.new(*([ self, next_job.split(',') ].flatten))
+      if first_job_from_requester?
+        fetch_code(job)
+        before_run(job) if File.exists?("#{job.project}/lib/tasks/testbot.rake")
+      end
+      
+      @instances << [ Thread.new { job.run(free_instance_number) },
+                      free_instance_number, job ]
+      @last_requester_mac = job.requester_mac
+      loop do
+        clear_completed_instances
+        break unless max_instances_running?
+      end
+    end
+  end
+  
+  def max_jruby_instances?
+    return unless @config.max_jruby_instances
+    @instances.find_all { |thread, n, job| job.jruby? }.size >= @config.max_jruby_instances
+  end
+  
+  def fetch_code(job)
+    system "rsync -az --delete -e ssh #{job.root}/ #{job.project}"
+  end
+  
+  def before_run(job)
+    system "export RAILS_ENV=test; export TEST_INSTANCES=#{@config.max_instances}; cd #{job.project}; rake testbot:before_run"
+  end
+  
+  def first_job_from_requester?
+    @last_requester_mac == nil
+  end
+  
+  def cpu_available?
+    @instances.size > 0 || CPU.current_usage < MAX_CPU_USAGE_WHEN_IDLE
+  end
+  
+  def time_for_update?
+    time_for_update = ((Time.now - @last_version_check) >= TIME_BETWEEN_VERSION_CHECKS)
+    @last_version_check = Time.now if time_for_update
+    time_for_update
+  end
+  
+  def check_for_update
+    return unless @config.auto_update
+    version = Server.get('/version') rescue Testbot::VERSION
+    return unless version != Testbot::VERSION
+
+    successful_install = system "gem install testbot -v #{version}"
+
+    # This closes the process and attempts to re-run the same command.
+    exec "testbot #{ARGV.join(' ')}" if successful_install
+  end
+  
+  def ping_params
+    { :hostname => (@hostname ||= `hostname`.chomp), :max_instances => @config.max_instances,
+      :idle_instances => (@config.max_instances - @instances.size), :username => ENV['USER'] }.merge(base_params)
+  end
+  
+  def base_params
+    { :version => Testbot::VERSION, :uid => @uid }
+  end
+  
+  def max_instances_running?
+    @instances.size == @config.max_instances
+  end
+
+  def clear_completed_instances
+    @instances.each_with_index do |data, index|
+      @instances.delete_at(index) if data.first.join(0.25)
+    end
+  end
+
+  def free_instance_number
+    0.upto(@config.max_instances - 1) do |number|
+      return number unless @instances.find { |instance, n, job| n == number }
+    end
+  end
+   
+  def start_ping
+    Thread.new do
+      while true
+        begin
+          Server.get("/runners/ping", :body => ping_params)
+        rescue
+        end
+        sleep TIME_BETWEEN_PINGS
+      end
+    end
+  end
+   
+end

data/lib/server.rb

@@ -0,0 +1,72 @@
+require 'rubygems'
+require 'sinatra'
+require 'yaml'
+require 'json'
+require File.join(File.dirname(__FILE__), 'server/job.rb')
+require File.join(File.dirname(__FILE__), 'server/group.rb') unless defined?(Group)
+require File.join(File.dirname(__FILE__), 'server/runner.rb')
+require File.join(File.dirname(__FILE__), 'server/build.rb')
+require File.expand_path(File.join(File.dirname(__FILE__), 'testbot'))
+
+if ENV['INTEGRATION_TEST']
+  set :port, 22880
+else
+  set :port, Testbot::SERVER_PORT
+end
+
+disable :logging if ENV['DISABLE_LOGGING']
+
+class Server
+  def self.valid_version?(runner_version)
+    Testbot::VERSION == runner_version
+  end
+end
+
+post '/builds' do
+  build = Build.create_and_build_jobs(params)[:id].to_s
+end
+
+get '/builds/:id' do
+  build = Build.find(:id => params[:id].to_i)
+  build.destroy if build[:done]
+  { "done" => build[:done], "results" => build[:results] }.to_json
+end
+
+get '/jobs/next' do
+  next_job, runner = Job.next(params, @env['REMOTE_ADDR'])
+  if next_job
+    next_job.update(:taken_at => Time.now, :taken_by_id => runner.id)
+    [ next_job[:id], next_job[:requester_mac], next_job[:project], next_job[:root], next_job[:type], (next_job[:jruby] == 1 ? 'jruby' : 'ruby'), next_job[:files] ].join(',')
+  end
+end
+
+put '/jobs/:id' do
+  Job.find(:id => params[:id].to_i).update(:result => params[:result]); nil
+end
+
+get '/runners/ping' do
+  return unless Server.valid_version?(params[:version])
+  runner = Runner.find(:uid => params[:uid])
+  runner.update(params.merge({ :last_seen_at => Time.now })) if runner
+  nil
+end
+
+get '/runners/outdated' do
+  Runner.find_all_outdated.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid] ].join(' ') }.join("\n").strip
+end
+
+get '/runners/available_instances' do
+  Runner.available_instances.to_s
+end
+
+get '/runners/total_instances' do
+  Runner.total_instances.to_s
+end
+
+get '/runners/available' do
+  Runner.find_all_available.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid], runner[:username], runner[:idle_instances] ].join(' ') }.join("\n").strip
+end
+
+get '/version' do
+  Testbot::VERSION
+end