testbot 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/server/build.rb CHANGED
@@ -1,29 +1,33 @@
1
- class Build < Sequel::Model
1
+ module Testbot::Server
2
2
 
3
- def self.create_and_build_jobs(hash)
4
- hash["jruby"] = (hash["jruby"] == "true") ? 1 : 0
5
- build = create(hash.reject { |k, v| k == 'available_runner_usage' })
6
- build.create_jobs!(hash['available_runner_usage'])
7
- build
8
- end
9
-
10
- def create_jobs!(available_runner_usage)
11
- groups = Group.build(self[:files].split, self[:sizes].split.map { |size| size.to_i },
12
- Runner.total_instances.to_f * (available_runner_usage.to_i / 100.0), self[:type])
13
- groups.each do |group|
14
- Job.create(:files => group.join(' '),
15
- :root => self[:root],
16
- :project => self[:project],
17
- :type => self[:type],
18
- :requester_mac => self[:requester_mac],
19
- :build_id => self[:id],
20
- :jruby => self[:jruby])
3
+ class Build < Sequel::Model
4
+
5
+ def self.create_and_build_jobs(hash)
6
+ hash["jruby"] = (hash["jruby"] == "true") ? 1 : 0
7
+ build = create(hash.reject { |k, v| k == 'available_runner_usage' })
8
+ build.create_jobs!(hash['available_runner_usage'])
9
+ build
21
10
  end
11
+
12
+ def create_jobs!(available_runner_usage)
13
+ groups = Group.build(self[:files].split, self[:sizes].split.map { |size| size.to_i },
14
+ Runner.total_instances.to_f * (available_runner_usage.to_i / 100.0), self[:type])
15
+ groups.each do |group|
16
+ Job.create(:files => group.join(' '),
17
+ :root => self[:root],
18
+ :project => self[:project],
19
+ :type => self[:type],
20
+ :requester_mac => self[:requester_mac],
21
+ :build_id => self[:id],
22
+ :jruby => self[:jruby])
23
+ end
24
+ end
25
+
26
+ def destroy
27
+ Job.filter([ 'build_id = ?', self[:id] ]).each { |job| job.destroy }
28
+ super
29
+ end
30
+
22
31
  end
23
-
24
- def destroy
25
- Job.filter([ 'build_id = ?', self[:id] ]).each { |job| job.destroy }
26
- super
27
- end
28
-
32
+
29
33
  end
data/lib/server/db.rb CHANGED
@@ -1,46 +1,49 @@
1
1
  require 'sequel'
2
2
 
3
- DB = Sequel.sqlite
3
+ module Testbot::Server
4
4
 
5
- DB.create_table :builds do
6
- primary_key :id
7
- String :files
8
- String :sizes
9
- String :results, :default => ''
10
- String :root
11
- String :project
12
- String :type
13
- String :requester_mac
14
- Integer :jruby
15
- Boolean :success, :default => true
16
- Boolean :done, :default => false
17
- end
5
+ DB = Sequel.sqlite
18
6
 
7
+ DB.create_table :builds do
8
+ primary_key :id
9
+ String :files
10
+ String :sizes
11
+ String :results, :default => ''
12
+ String :root
13
+ String :project
14
+ String :type
15
+ String :requester_mac
16
+ Integer :jruby
17
+ Boolean :success, :default => true
18
+ Boolean :done, :default => false
19
+ end
19
20
 
20
- DB.create_table :jobs do
21
- primary_key :id
22
- String :files
23
- String :result
24
- String :root
25
- String :project
26
- String :type
27
- String :requester_mac
28
- Integer :jruby
29
- Integer :build_id
30
- Integer :taken_by_id
31
- Boolean :success
32
- Datetime :taken_at, :default => nil
33
- end
34
21
 
35
- DB.create_table :runners do
36
- primary_key :id
37
- String :ip
38
- String :hostname
39
- String :uid
40
- String :username
41
- String :version
42
- Integer :idle_instances
43
- Integer :max_instances
44
- Datetime :last_seen_at
45
- end
22
+ DB.create_table :jobs do
23
+ primary_key :id
24
+ String :files
25
+ String :result
26
+ String :root
27
+ String :project
28
+ String :type
29
+ String :requester_mac
30
+ Integer :jruby
31
+ Integer :build_id
32
+ Integer :taken_by_id
33
+ Boolean :success
34
+ Datetime :taken_at, :default => nil
35
+ end
46
36
 
37
+ DB.create_table :runners do
38
+ primary_key :id
39
+ String :ip
40
+ String :hostname
41
+ String :uid
42
+ String :username
43
+ String :version
44
+ Integer :idle_instances
45
+ Integer :max_instances
46
+ Datetime :last_seen_at
47
+ end
48
+
49
+ end
data/lib/server/group.rb CHANGED
@@ -1,44 +1,48 @@
1
1
  require 'rubygems'
2
2
 
3
- class Group
4
-
5
- DEFAULT = nil
6
-
7
- def self.build(files, sizes, instance_count, type)
8
- tests_with_sizes = slow_tests_first(map_files_and_sizes(files, sizes))
9
-
10
- groups = []
11
- current_group, current_size = 0, 0
12
- tests_with_sizes.each do |test, size|
13
- # inserts into next group if current is full and we are not in the last group
14
- if (0.5*size + current_size) > group_size(tests_with_sizes, instance_count) and instance_count > current_group + 1
15
- current_size = size
16
- current_group += 1
17
- else
18
- current_size += size
3
+ module Testbot::Server
4
+
5
+ class Group
6
+
7
+ DEFAULT = nil
8
+
9
+ def self.build(files, sizes, instance_count, type)
10
+ tests_with_sizes = slow_tests_first(map_files_and_sizes(files, sizes))
11
+
12
+ groups = []
13
+ current_group, current_size = 0, 0
14
+ tests_with_sizes.each do |test, size|
15
+ # inserts into next group if current is full and we are not in the last group
16
+ if (0.5*size + current_size) > group_size(tests_with_sizes, instance_count) and instance_count > current_group + 1
17
+ current_size = size
18
+ current_group += 1
19
+ else
20
+ current_size += size
21
+ end
22
+ groups[current_group] ||= []
23
+ groups[current_group] << test
19
24
  end
20
- groups[current_group] ||= []
21
- groups[current_group] << test
25
+
26
+ groups.compact
22
27
  end
23
-
24
- groups.compact
25
- end
26
-
27
- private
28
-
29
- def self.group_size(tests_with_sizes, group_count)
30
- total = tests_with_sizes.inject(0) { |sum, test| sum += test[1] }
31
- total / group_count.to_f
32
- end
33
-
34
- def self.map_files_and_sizes(files, sizes)
35
- list = []
36
- files.each_with_index { |file, i| list << [ file, sizes[i] ] }
37
- list
38
- end
39
-
40
- def self.slow_tests_first(tests)
41
- tests.sort_by { |test, time| time.to_i }.reverse
28
+
29
+ private
30
+
31
+ def self.group_size(tests_with_sizes, group_count)
32
+ total = tests_with_sizes.inject(0) { |sum, test| sum += test[1] }
33
+ total / group_count.to_f
34
+ end
35
+
36
+ def self.map_files_and_sizes(files, sizes)
37
+ list = []
38
+ files.each_with_index { |file, i| list << [ file, sizes[i] ] }
39
+ list
40
+ end
41
+
42
+ def self.slow_tests_first(tests)
43
+ tests.sort_by { |test, time| time.to_i }.reverse
44
+ end
45
+
42
46
  end
43
47
 
44
- end
48
+ end
data/lib/server/job.rb CHANGED
@@ -1,44 +1,48 @@
1
- require File.join(File.dirname(__FILE__), 'db.rb') unless defined?(DB)
2
-
3
- class Job < Sequel::Model
4
- def update(hash)
5
- super(hash)
6
- if build = Build.find([ "id = ?", self[:build_id] ])
7
- done = Job.filter([ "result IS NULL AND build_id = ?", self[:build_id] ]).count == 0
8
- build.update(:results => build[:results].to_s + hash[:result].to_s,
9
- :done => done)
10
-
11
- build_broken_by_job = (hash[:success] == "false" && build[:success])
12
- build.update(:success => false) if build_broken_by_job
1
+ require File.expand_path(File.join(File.dirname(__FILE__), 'db.rb'))
2
+
3
+ module Testbot::Server
4
+
5
+ class Job < Sequel::Model
6
+ def update(hash)
7
+ super(hash)
8
+ if build = Build.find([ "id = ?", self[:build_id] ])
9
+ done = Job.filter([ "result IS NULL AND build_id = ?", self[:build_id] ]).count == 0
10
+ build.update(:results => build[:results].to_s + hash[:result].to_s,
11
+ :done => done)
12
+
13
+ build_broken_by_job = (hash[:success] == "false" && build[:success])
14
+ build.update(:success => false) if build_broken_by_job
15
+ end
13
16
  end
14
- end
15
-
16
- def self.next(params, remove_addr)
17
- clean_params = params.reject { |k, v| [ "requester_mac", "no_jruby" ].include?(k) }
18
- runner = Runner.record! clean_params.merge({ :ip => remove_addr, :last_seen_at => Time.now })
19
- return unless Server.valid_version?(params[:version])
20
- [ next_job_query(params["requester_mac"], params["no_jruby"]).first, runner ]
21
- end
22
-
23
- private
24
-
25
- def self.next_job_query(requester_mac, no_jruby)
26
- release_jobs_taken_by_missing_runners!
27
- query = Job.filter("taken_at IS NULL").order("Random()".lit)
28
- filters = []
29
- filters << "requester_mac = '#{requester_mac}'" if requester_mac
30
- filters << "jruby != 1" if no_jruby
31
- if filters.empty?
32
- query
33
- else
34
- query.filter(filters.join(' AND '))
17
+
18
+ def self.next(params, remove_addr)
19
+ clean_params = params.reject { |k, v| [ "requester_mac", "no_jruby" ].include?(k) }
20
+ runner = Runner.record! clean_params.merge({ :ip => remove_addr, :last_seen_at => Time.now })
21
+ return unless Server.valid_version?(params[:version])
22
+ [ next_job_query(params["requester_mac"], params["no_jruby"]).first, runner ]
23
+ end
24
+
25
+ private
26
+
27
+ def self.next_job_query(requester_mac, no_jruby)
28
+ release_jobs_taken_by_missing_runners!
29
+ query = Job.filter("taken_at IS NULL").order("Random()".lit)
30
+ filters = []
31
+ filters << "requester_mac = '#{requester_mac}'" if requester_mac
32
+ filters << "jruby != 1" if no_jruby
33
+ if filters.empty?
34
+ query
35
+ else
36
+ query.filter(filters.join(' AND '))
37
+ end
38
+ end
39
+
40
+ def self.release_jobs_taken_by_missing_runners!
41
+ missing_runners = Runner.filter([ "last_seen_at < ?", (Time.now - Runner.timeout) ])
42
+ missing_runners.each { |r|
43
+ Job.filter(:taken_by_id => r[:id]).update(:taken_at => nil)
44
+ }
35
45
  end
36
46
  end
37
-
38
- def self.release_jobs_taken_by_missing_runners!
39
- missing_runners = Runner.filter([ "last_seen_at < ?", (Time.now - Runner.timeout) ])
40
- missing_runners.each { |r|
41
- Job.filter(:taken_by_id => r[:id]).update(:taken_at => nil)
42
- }
43
- end
47
+
44
48
  end
data/lib/server/runner.rb CHANGED
@@ -1,38 +1,42 @@
1
- require File.join(File.dirname(__FILE__), 'db.rb') unless defined?(DB)
1
+ require File.expand_path(File.join(File.dirname(__FILE__), 'db.rb'))
2
2
 
3
- class Runner < Sequel::Model
3
+ module Testbot::Server
4
4
 
5
- def self.record!(hash)
6
- create_or_update_by_mac!(hash)
7
- end
8
-
9
- def self.create_or_update_by_mac!(hash)
10
- if (runner = find(:uid => hash[:uid]))
11
- runner.update hash
12
- else
13
- Runner.create hash
5
+ class Runner < Sequel::Model
6
+
7
+ def self.record!(hash)
8
+ create_or_update_by_mac!(hash)
14
9
  end
10
+
11
+ def self.create_or_update_by_mac!(hash)
12
+ if (runner = find(:uid => hash[:uid]))
13
+ runner.update hash
14
+ else
15
+ Runner.create hash
16
+ end
17
+ end
18
+
19
+ def self.timeout
20
+ 10
21
+ end
22
+
23
+ def self.find_all_outdated
24
+ DB[:runners].filter("version != ? OR version IS NULL", Testbot.version)
25
+ end
26
+
27
+ def self.find_all_available
28
+ DB[:runners].filter("version = ? AND last_seen_at > ?", Testbot.version, Time.now - Runner.timeout)
29
+ end
30
+
31
+ def self.available_instances
32
+ find_all_available.inject(0) { |sum, r| r[:idle_instances] + sum }
33
+ end
34
+
35
+ def self.total_instances
36
+ return 1 if ENV['INTEGRATION_TEST']
37
+ find_all_available.inject(0) { |sum, r| r[:max_instances] + sum }
38
+ end
39
+
15
40
  end
16
-
17
- def self.timeout
18
- 10
19
- end
20
-
21
- def self.find_all_outdated
22
- DB[:runners].filter("version != ? OR version IS NULL", Testbot.version)
23
- end
24
-
25
- def self.find_all_available
26
- DB[:runners].filter("version = ? AND last_seen_at > ?", Testbot.version, Time.now - Runner.timeout)
27
- end
28
-
29
- def self.available_instances
30
- find_all_available.inject(0) { |sum, r| r[:idle_instances] + sum }
31
- end
32
-
33
- def self.total_instances
34
- return 1 if ENV['INTEGRATION_TEST']
35
- find_all_available.inject(0) { |sum, r| r[:max_instances] + sum }
36
- end
37
-
41
+
38
42
  end
@@ -0,0 +1,74 @@
1
+ require 'rubygems'
2
+ require 'sinatra'
3
+ require 'yaml'
4
+ require 'json'
5
+ require File.expand_path(File.join(File.dirname(__FILE__), '/../shared/testbot'))
6
+ require File.expand_path(File.join(File.dirname(__FILE__), 'job.rb'))
7
+ require File.expand_path(File.join(File.dirname(__FILE__), 'group.rb')) #unless defined?(Group)
8
+ require File.expand_path(File.join(File.dirname(__FILE__), 'runner.rb'))
9
+ require File.expand_path(File.join(File.dirname(__FILE__), 'build.rb'))
10
+
11
+ module Testbot::Server
12
+
13
+ if ENV['INTEGRATION_TEST']
14
+ set :port, 22880
15
+ else
16
+ set :port, Testbot::SERVER_PORT
17
+ end
18
+
19
+ class Server
20
+ def self.valid_version?(runner_version)
21
+ Testbot.version == runner_version
22
+ end
23
+ end
24
+
25
+ post '/builds' do
26
+ build = Build.create_and_build_jobs(params)[:id].to_s
27
+ end
28
+
29
+ get '/builds/:id' do
30
+ build = Build.find(:id => params[:id].to_i)
31
+ build.destroy if build[:done]
32
+ { "done" => build[:done], "results" => build[:results], "success" => build[:success] }.to_json
33
+ end
34
+
35
+ get '/jobs/next' do
36
+ next_job, runner = Job.next(params, @env['REMOTE_ADDR'])
37
+ if next_job
38
+ next_job.update(:taken_at => Time.now, :taken_by_id => runner.id)
39
+ [ next_job[:id], next_job[:requester_mac], next_job[:project], next_job[:root], next_job[:type], (next_job[:jruby] == 1 ? 'jruby' : 'ruby'), next_job[:files] ].join(',')
40
+ end
41
+ end
42
+
43
+ put '/jobs/:id' do
44
+ Job.find(:id => params[:id].to_i).update(:result => params[:result], :success => params[:success]); nil
45
+ end
46
+
47
+ get '/runners/ping' do
48
+ return unless Server.valid_version?(params[:version])
49
+ runner = Runner.find(:uid => params[:uid])
50
+ runner.update(params.merge({ :last_seen_at => Time.now })) if runner
51
+ nil
52
+ end
53
+
54
+ get '/runners/outdated' do
55
+ Runner.find_all_outdated.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid] ].join(' ') }.join("\n").strip
56
+ end
57
+
58
+ get '/runners/available_instances' do
59
+ Runner.available_instances.to_s
60
+ end
61
+
62
+ get '/runners/total_instances' do
63
+ Runner.total_instances.to_s
64
+ end
65
+
66
+ get '/runners/available' do
67
+ Runner.find_all_available.map { |runner| [ runner[:ip], runner[:hostname], runner[:uid], runner[:username], runner[:idle_instances] ].join(' ') }.join("\n").strip
68
+ end
69
+
70
+ get '/version' do
71
+ Testbot.version
72
+ end
73
+
74
+ end