testbot 0.4.6 → 0.4.7

data/lib/server/build.rb

@@ -1,29 +1,33 @@
-class Build < Sequel::Model
+module Testbot::Server
 
-  def self.create_and_build_jobs(hash)
-    hash["jruby"] = (hash["jruby"] == "true") ? 1 : 0
-    build = create(hash.reject { |k, v| k == 'available_runner_usage' })
-    build.create_jobs!(hash['available_runner_usage'])
-    build
-  end
-  
-  def create_jobs!(available_runner_usage)
-    groups = Group.build(self[:files].split, self[:sizes].split.map { |size| size.to_i },
-                     Runner.total_instances.to_f * (available_runner_usage.to_i / 100.0), self[:type])
-    groups.each do |group|
-      Job.create(:files => group.join(' '),
-                 :root => self[:root],
-                 :project => self[:project],
-                 :type => self[:type],
-                 :requester_mac => self[:requester_mac],
-                 :build_id => self[:id],
-                 :jruby => self[:jruby])
+  class Build < Sequel::Model
+
+    def self.create_and_build_jobs(hash)
+      hash["jruby"] = (hash["jruby"] == "true") ? 1 : 0
+      build = create(hash.reject { |k, v| k == 'available_runner_usage' })
+      build.create_jobs!(hash['available_runner_usage'])
+      build
     end
+
+    def create_jobs!(available_runner_usage)
+      groups = Group.build(self[:files].split, self[:sizes].split.map { |size| size.to_i },
+                           Runner.total_instances.to_f * (available_runner_usage.to_i / 100.0), self[:type])
+      groups.each do |group|
+        Job.create(:files => group.join(' '),
+                   :root => self[:root],
+                   :project => self[:project],
+                   :type => self[:type],
+                   :requester_mac => self[:requester_mac],
+                   :build_id => self[:id],
+                   :jruby => self[:jruby])
+      end
+    end
+
+    def destroy
+      Job.filter([ 'build_id = ?', self[:id] ]).each { |job| job.destroy }
+      super
+    end
+
   end
-  
-  def destroy
-    Job.filter([ 'build_id = ?', self[:id] ]).each { |job| job.destroy }
-    super
-  end
-  
+
 end

data/lib/server/db.rb

@@ -1,46 +1,49 @@
 require 'sequel'
 
-DB = Sequel.sqlite
+module Testbot::Server
 
-DB.create_table :builds do
-  primary_key :id
-  String :files
-  String :sizes
-  String :results, :default => ''
-  String :root
-  String :project
-  String :type
-  String :requester_mac
-  Integer :jruby
-  Boolean :success, :default => true
-  Boolean :done, :default => false
-end
+  DB = Sequel.sqlite
 
+  DB.create_table :builds do
+    primary_key :id
+    String :files
+    String :sizes
+    String :results, :default => ''
+    String :root
+    String :project
+    String :type
+    String :requester_mac
+    Integer :jruby
+    Boolean :success, :default => true
+    Boolean :done, :default => false
+  end
 
-DB.create_table :jobs do
-  primary_key :id
-  String :files
-  String :result
-  String :root
-  String :project
-  String :type
-  String :requester_mac
-  Integer :jruby
-  Integer :build_id
-  Integer :taken_by_id
-  Boolean :success
-  Datetime :taken_at, :default => nil
-end
 
-DB.create_table :runners do
-  primary_key :id
-  String :ip
-  String :hostname
-  String :uid
-  String :username
-  String :version
-  Integer :idle_instances
-  Integer :max_instances
-  Datetime :last_seen_at
-end
+  DB.create_table :jobs do
+    primary_key :id
+    String :files
+    String :result
+    String :root
+    String :project
+    String :type
+    String :requester_mac
+    Integer :jruby
+    Integer :build_id
+    Integer :taken_by_id
+    Boolean :success
+    Datetime :taken_at, :default => nil
+  end
 
+  DB.create_table :runners do
+    primary_key :id
+    String :ip
+    String :hostname
+    String :uid
+    String :username
+    String :version
+    Integer :idle_instances
+    Integer :max_instances
+    Datetime :last_seen_at
+  end
+
+end

data/lib/server/group.rb

@@ -1,44 +1,48 @@
 require 'rubygems'
 
-class Group
-  
-  DEFAULT = nil
-
-  def self.build(files, sizes, instance_count, type)
-    tests_with_sizes = slow_tests_first(map_files_and_sizes(files, sizes))
-    
-    groups = []
-    current_group, current_size = 0, 0
-    tests_with_sizes.each do |test, size|
-      # inserts into next group if current is full and we are not in the last group
-      if (0.5*size + current_size) > group_size(tests_with_sizes, instance_count) and instance_count > current_group + 1
-        current_size = size
-        current_group += 1
-      else
-        current_size += size
+module Testbot::Server
+
+  class Group
+
+    DEFAULT = nil
+
+    def self.build(files, sizes, instance_count, type)
+      tests_with_sizes = slow_tests_first(map_files_and_sizes(files, sizes))
+
+      groups = []
+      current_group, current_size = 0, 0
+      tests_with_sizes.each do |test, size|
+        # inserts into next group if current is full and we are not in the last group
+        if (0.5*size + current_size) > group_size(tests_with_sizes, instance_count) and instance_count > current_group + 1
+          current_size = size
+          current_group += 1
+        else
+          current_size += size
+        end
+        groups[current_group] ||= []
+        groups[current_group] << test
       end
-      groups[current_group] ||= []
-      groups[current_group] << test
+
+      groups.compact
     end
-    
-    groups.compact
-  end
-    
-  private
-  
-  def self.group_size(tests_with_sizes, group_count)
-    total = tests_with_sizes.inject(0) { |sum, test| sum += test[1] }
-    total / group_count.to_f
-  end
-  
-  def self.map_files_and_sizes(files, sizes)
-    list = []
-    files.each_with_index { |file, i| list << [ file, sizes[i] ] }
-    list
-  end
-  
-  def self.slow_tests_first(tests)
-    tests.sort_by { |test, time| time.to_i }.reverse
+
+    private
+
+    def self.group_size(tests_with_sizes, group_count)
+      total = tests_with_sizes.inject(0) { |sum, test| sum += test[1] }
+      total / group_count.to_f
+    end
+
+    def self.map_files_and_sizes(files, sizes)
+      list = []
+      files.each_with_index { |file, i| list << [ file, sizes[i] ] }
+      list
+    end
+
+    def self.slow_tests_first(tests)
+      tests.sort_by { |test, time| time.to_i }.reverse
+    end
+
   end
 
-end
+end

data/lib/server/job.rb

@@ -1,44 +1,48 @@
-require File.join(File.dirname(__FILE__), 'db.rb') unless defined?(DB)
-
-class Job < Sequel::Model
-  def update(hash)
-    super(hash)
-    if build = Build.find([ "id = ?", self[:build_id] ])
-      done = Job.filter([ "result IS NULL AND build_id = ?", self[:build_id] ]).count == 0
-      build.update(:results => build[:results].to_s + hash[:result].to_s,
-                   :done => done)
-
-      build_broken_by_job = (hash[:success] == "false" && build[:success])
-      build.update(:success => false) if build_broken_by_job
+require File.expand_path(File.join(File.dirname(__FILE__), 'db.rb'))
+
+module Testbot::Server
+
+  class Job < Sequel::Model
+    def update(hash)
+      super(hash)
+      if build = Build.find([ "id = ?", self[:build_id] ])
+        done = Job.filter([ "result IS NULL AND build_id = ?", self[:build_id] ]).count == 0
+        build.update(:results => build[:results].to_s + hash[:result].to_s,
+                     :done => done)
+
+        build_broken_by_job = (hash[:success] == "false" && build[:success])
+        build.update(:success => false) if build_broken_by_job
+      end
     end
-  end
-  
-  def self.next(params, remove_addr)
-    clean_params = params.reject { |k, v| [ "requester_mac", "no_jruby" ].include?(k) }
-    runner = Runner.record! clean_params.merge({ :ip => remove_addr, :last_seen_at => Time.now })
-    return unless Server.valid_version?(params[:version])
-    [ next_job_query(params["requester_mac"], params["no_jruby"]).first, runner ]
-  end
-  
-  private
-  
-  def self.next_job_query(requester_mac, no_jruby)
-    release_jobs_taken_by_missing_runners!
-    query = Job.filter("taken_at IS NULL").order("Random()".lit)
-    filters = []
-    filters << "requester_mac = '#{requester_mac}'" if requester_mac
-    filters << "jruby != 1" if no_jruby
-    if filters.empty?
-      query
-    else
-      query.filter(filters.join(' AND ')) 
+
+    def self.next(params, remove_addr)
+      clean_params = params.reject { |k, v| [ "requester_mac", "no_jruby" ].include?(k) }
+      runner = Runner.record! clean_params.merge({ :ip => remove_addr, :last_seen_at => Time.now })
+      return unless Server.valid_version?(params[:version])
+      [ next_job_query(params["requester_mac"], params["no_jruby"]).first, runner ]
+    end
+
+    private
+
+    def self.next_job_query(requester_mac, no_jruby)
+      release_jobs_taken_by_missing_runners!
+      query = Job.filter("taken_at IS NULL").order("Random()".lit)
+      filters = []
+      filters << "requester_mac = '#{requester_mac}'" if requester_mac
+      filters << "jruby != 1" if no_jruby
+      if filters.empty?
+        query
+      else
+        query.filter(filters.join(' AND ')) 
+      end
+    end
+
+    def self.release_jobs_taken_by_missing_runners!
+      missing_runners = Runner.filter([ "last_seen_at < ?", (Time.now - Runner.timeout) ])
+      missing_runners.each { |r|
+        Job.filter(:taken_by_id => r[:id]).update(:taken_at => nil)
+      }    
     end
   end
-  
-  def self.release_jobs_taken_by_missing_runners!
-    missing_runners = Runner.filter([ "last_seen_at < ?", (Time.now - Runner.timeout) ])
-    missing_runners.each { |r|
-      Job.filter(:taken_by_id => r[:id]).update(:taken_at => nil)
-    }    
-  end
+
 end

data/lib/server/runner.rb

@@ -1,38 +1,42 @@
-require File.join(File.dirname(__FILE__), 'db.rb') unless defined?(DB)
+require File.expand_path(File.join(File.dirname(__FILE__), 'db.rb'))
 
-class Runner < Sequel::Model
+module Testbot::Server
 
-  def self.record!(hash)
-    create_or_update_by_mac!(hash)
-  end
-  
-  def self.create_or_update_by_mac!(hash)
-    if (runner = find(:uid => hash[:uid]))
-      runner.update hash
-    else
-      Runner.create hash
+  class Runner < Sequel::Model
+
+    def self.record!(hash)
+      create_or_update_by_mac!(hash)
     end
+
+    def self.create_or_update_by_mac!(hash)
+      if (runner = find(:uid => hash[:uid]))
+        runner.update hash
+      else
+        Runner.create hash
+      end
+    end
+
+    def self.timeout
+      10
+    end
+
+    def self.find_all_outdated
+      DB[:runners].filter("version != ? OR version IS NULL", Testbot.version)
+    end
+
+    def self.find_all_available
+      DB[:runners].filter("version = ? AND last_seen_at > ?", Testbot.version, Time.now - Runner.timeout)
+    end  
+
+    def self.available_instances
+      find_all_available.inject(0) { |sum, r| r[:idle_instances] + sum }
+    end
+
+    def self.total_instances
+      return 1 if ENV['INTEGRATION_TEST']
+      find_all_available.inject(0) { |sum, r| r[:max_instances] + sum }
+    end
+
   end
-  
-  def self.timeout
-    10
-  end
-  
-  def self.find_all_outdated
-    DB[:runners].filter("version != ? OR version IS NULL", Testbot.version)
-  end
-  
-  def self.find_all_available
-    DB[:runners].filter("version = ? AND last_seen_at > ?", Testbot.version, Time.now - Runner.timeout)
-  end  
-  
-  def self.available_instances
-    find_all_available.inject(0) { |sum, r| r[:idle_instances] + sum }
-  end
-  
-  def self.total_instances
-    return 1 if ENV['INTEGRATION_TEST']
-    find_all_available.inject(0) { |sum, r| r[:max_instances] + sum }
-  end
-  
+
 end

data/lib/server/server.rb

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'sinatra'
+require 'yaml'
+require 'json'
+require File.expand_path(File.join(File.dirname(__FILE__), '/../shared/testbot'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'job.rb'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'group.rb')) #unless defined?(Group)
+require File.expand_path(File.join(File.dirname(__FILE__), 'runner.rb'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'build.rb'))
+
+module Testbot::Server
+
+  if ENV['INTEGRATION_TEST']
+    set :port, 22880
+  else
+    set :port, Testbot::SERVER_PORT
+  end
+
+  class Server
+    def self.valid_version?(runner_version)
+      Testbot.version == runner_version
+    end
+  end
+
+  post '/builds' do
+    build = Build.create_and_build_jobs(params)[:id].to_s
+  end
+
+  get '/builds/:id' do
+    build = Build.find(:id => params[:id].to_i)
+    build.destroy if build[:done]
+    { "done" => build[:done], "results" => build[:results], "success" => build[:success] }.to_json
+  end
+
+  get '/jobs/next' do
+    next_job, runner = Job.next(params, @env['REMOTE_ADDR'])
+    if next_job
+      next_job.update(:taken_at => Time.now, :taken_by_id => runner.id)
+      [ next_job[:id], next_job[:requester_mac], next_job[:project], next_job[:root], next_job[:type], (next_job[:jruby] == 1 ? 'jruby' : 'ruby'), next_job[:files] ].join(',')
+    end
+  end
+
+  put '/jobs/:id' do
+    Job.find(:id => params[:id].to_i).update(:result => params[:result], :success => params[:success]); nil
+  end
+
+  get '/runners/ping' do
+    return unless Server.valid_version?(params[:version])
+    runner = Runner.find(:uid => params[:uid])
+    runner.update(params.merge({ :last_seen_at => Time.now })) if runner
+    nil
+  end
+
+  get '/runners/outdated' do
+    Runner.find_all_outdated.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid] ].join(' ') }.join("\n").strip
+  end
+
+  get '/runners/available_instances' do
+    Runner.available_instances.to_s
+  end
+
+  get '/runners/total_instances' do
+    Runner.total_instances.to_s
+  end
+
+  get '/runners/available' do
+    Runner.find_all_available.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid], runner[:username], runner[:idle_instances] ].join(' ') }.join("\n").strip
+  end
+
+  get '/version' do
+    Testbot.version
+  end
+
+end