rrrspec-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,111 @@
1
+ module RRRSpec
2
+ module Server
3
+ module Persistence
4
+ class Taskset < ActiveRecord::Base
5
+ include ActiveModel::Serializers::JSON
6
+
7
+ has_many :worker_logs
8
+ has_many :slaves
9
+ has_many :tasks
10
+
11
+ scope :full, includes(
12
+ :tasks => [{:trials => [:task, :slave]}, :taskset],
13
+ :slaves => [:trials],
14
+ :worker_logs => [:taskset]
15
+ )
16
+
17
+ def as_nodetail_json
18
+ as_json(except: :id)
19
+ end
20
+
21
+ def as_short_json
22
+ h = as_json(except: :id)
23
+ h['slaves'] = slaves.map { |slave| slave.as_json(only: :key) }
24
+ h['tasks'] = tasks.map { |task| task.as_json(only: :key) }
25
+ h['worker_logs'] = worker_logs.map { |worker_log| worker_log.as_json(only: :key) }
26
+ h
27
+ end
28
+
29
+ def as_full_json
30
+ h = as_json(except: :id)
31
+ h['slaves'] = slaves.map(&:as_full_json)
32
+ h['tasks'] = tasks.map(&:as_full_json)
33
+ h['worker_logs'] = worker_logs.map(&:as_full_json)
34
+ h
35
+ end
36
+ end
37
+
38
+ class Task < ActiveRecord::Base
39
+ include ActiveModel::Serializers::JSON
40
+
41
+ belongs_to :taskset
42
+ has_many :trials
43
+
44
+ def as_short_json
45
+ h = as_json(except: [:id, :taskset_id, :trials],
46
+ include: { 'taskset' => { only: :key } })
47
+ h['trials'] = trials.map { |trial| trial.as_json(only: :key) }
48
+ h
49
+ end
50
+
51
+ def as_full_json
52
+ h = as_json(except: [:id, :taskset_id, :trials],
53
+ include: { 'taskset' => { only: :key } })
54
+ h['trials'] = trials.map(&:as_full_json)
55
+ h
56
+ end
57
+ end
58
+
59
+ class Trial < ActiveRecord::Base
60
+ include ActiveModel::Serializers::JSON
61
+
62
+ belongs_to :task
63
+ belongs_to :slave
64
+
65
+ def as_full_json
66
+ as_json(except: [:id, :task_id, :slave_id],
67
+ include: { 'slave' => { only: :key }, 'task' => { only: :key } })
68
+ end
69
+
70
+ def as_short_json
71
+ as_full_json
72
+ end
73
+ end
74
+
75
+ class WorkerLog < ActiveRecord::Base
76
+ include ActiveModel::Serializers::JSON
77
+
78
+ belongs_to :taskset
79
+
80
+ def as_full_json
81
+ as_json(except: [:id, :taskset_id, :worker_key],
82
+ include: { 'taskset' => { only: :key } },
83
+ methods: :worker)
84
+ end
85
+
86
+ def as_short_json
87
+ as_full_json
88
+ end
89
+
90
+ def worker
91
+ { 'key' => worker_key }
92
+ end
93
+ end
94
+
95
+ class Slave < ActiveRecord::Base
96
+ include ActiveModel::Serializers::JSON
97
+
98
+ has_many :trials
99
+
100
+ def as_full_json
101
+ as_json(except: [:id, :taskset_id],
102
+ include: { 'trials' => { only: :key } })
103
+ end
104
+
105
+ def as_short_json
106
+ as_full_json
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,147 @@
1
+ require 'zlib'
2
+ require 'activerecord-import'
3
+ require 'active_support/inflector'
4
+ ActiveSupport::Inflector::Inflections.instance.singular('Slaves', 'Slave')
5
+ ActiveSupport::Inflector::Inflections.instance.singular('slaves', 'slave')
6
+ ActiveRecord::Base.include_root_in_json = false
7
+ ActiveRecord::Base.default_timezone = :utc
8
+
9
+ module RRRSpec
10
+ module Server
11
+ module Persister
12
+ SLAVE_EXIT_WAIT_TIME = 15
13
+ PERSISTED_RESIDUE_SEC = 60
14
+
15
+ module_function
16
+
17
+ def work_loop
18
+ loop { work }
19
+ end
20
+
21
+ def work
22
+ taskset = PersisterQueue.dequeue
23
+ ActiveRecord::Base.connection_pool.with_connection do
24
+ return if Persistence::Taskset.where(key: taskset.key).exists?
25
+ end
26
+
27
+ sleep SLAVE_EXIT_WAIT_TIME
28
+
29
+ ActiveRecord::Base.connection_pool.with_connection do
30
+ persist(taskset)
31
+ if RRRSpec.configuration.json_cache_path
32
+ create_api_cache(taskset, RRRSpec.configuration.json_cache_path)
33
+ end
34
+ taskset.expire(PERSISTED_RESIDUE_SEC)
35
+ update_estimate_sec(taskset)
36
+ end
37
+ rescue
38
+ RRRSpec.logger.error($!)
39
+ end
40
+
41
+ private
42
+ module_function
43
+
44
+ def persist(taskset)
45
+ taskset_finished_at = taskset.finished_at
46
+ return if taskset_finished_at.blank?
47
+
48
+ p_taskset = ActiveRecord::Base.transaction do
49
+ h = taskset.to_h
50
+ h.delete('tasks')
51
+ h.delete('slaves')
52
+ h.delete('worker_logs')
53
+ Persistence::Taskset.create(h)
54
+ end
55
+
56
+ ActiveRecord::Base.transaction do
57
+ p_slaves = taskset.slaves.map do |slave|
58
+ h = slave.to_h
59
+ h.delete('trials')
60
+ p_slave = Persistence::Slave.new(h)
61
+ p_slave.taskset_id = p_taskset.id
62
+ p_slave
63
+ end
64
+ Persistence::Slave.import(p_slaves)
65
+ end
66
+
67
+ ActiveRecord::Base.transaction do
68
+ Persistence::Task.import(taskset.tasks.map do |task|
69
+ h = task.to_h
70
+ h.delete('taskset')
71
+ h.delete('trials')
72
+ p_task = Persistence::Task.new(h)
73
+ p_task.taskset_id = p_taskset
74
+ p_task
75
+ end)
76
+ end
77
+
78
+ p_slaves = {}
79
+ p_taskset.slaves.each do |p_slave|
80
+ p_slaves[p_slave.key] = p_slave
81
+ end
82
+
83
+ ActiveRecord::Base.transaction do
84
+ p_trials = []
85
+ p_taskset.tasks.each do |p_task|
86
+ Task.new(p_task.key).trials.each do |trial|
87
+ h = trial.to_h
88
+ next if h['finished_at'].blank? || h['finished_at'] > taskset_finished_at
89
+ slave_key = h.delete('slave')['key']
90
+ h.delete('task')
91
+ p_trial = Persistence::Trial.new(h)
92
+ p_trial.task_id = p_task
93
+ p_trial.slave_id = p_slaves[slave_key]
94
+ p_trials << p_trial
95
+ end
96
+ end
97
+ Persistence::Trial.import(p_trials)
98
+ end
99
+
100
+ ActiveRecord::Base.transaction do
101
+ Persistence::WorkerLog.import(taskset.worker_logs.map do |worker_log|
102
+ h = worker_log.to_h
103
+ h['worker_key'] = h['worker']['key']
104
+ h.delete('worker')
105
+ h.delete('taskset')
106
+ p_worker_log = Persistence::WorkerLog.new(h)
107
+ p_worker_log.taskset_id = p_taskset
108
+ p_worker_log
109
+ end)
110
+ end
111
+ end
112
+
113
+ def create_api_cache(taskset, path)
114
+ p_obj = Persistence::Taskset.where(key: taskset.key).full.first
115
+ json = JSON.generate(p_obj.as_full_json.update('is_full' => true))
116
+
117
+ FileUtils.mkdir_p(File.join(path, 'v1', 'tasksets'))
118
+ json_path = File.join(path, 'v1', 'tasksets', taskset.key.gsub(':', '-'))
119
+ IO.write(json_path, json)
120
+ Zlib::GzipWriter.open(json_path + ".gz") { |gz| gz.write(json) }
121
+ end
122
+
123
+ ESTIMATION_FIELDS = [
124
+ "`spec_file`",
125
+ "avg(UNIX_TIMESTAMP(`trials`.`finished_at`)-UNIX_TIMESTAMP(`trials`.`started_at`)) as `avg`",
126
+ # "avg(`trials`.`finished_at`-`trials`.`started_at`) as `avg`",
127
+ ]
128
+
129
+ def update_estimate_sec(taskset)
130
+ p_obj = Persistence::Taskset.where(key: taskset.key).first
131
+ taskset_class = p_obj.taskset_class
132
+ query = Persistence::Task.joins(:trials).joins(:taskset).
133
+ select(ESTIMATION_FIELDS).
134
+ where('tasksets.taskset_class' => taskset_class).
135
+ where('trials.status' => ["passed", "pending"]).
136
+ group('spec_file')
137
+ estimation = {}
138
+ query.each do |row|
139
+ estimation[row.spec_file] = row.avg.to_i
140
+ end
141
+ unless estimation.empty?
142
+ TasksetEstimation.update_estimate_secs(taskset_class, estimation)
143
+ end
144
+ end
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,5 @@
1
+ module RRRSpec
2
+ module Server
3
+ VERSION = "0.2.0"
4
+ end
5
+ end
@@ -0,0 +1,245 @@
1
+ module RRRSpec
2
+ module Server
3
+ class WorkerRunner
4
+ CANCEL_POLLING = 10
5
+ TIMEOUT_EXITCODE = 42
6
+
7
+ attr_reader :internal_status, :current_taskset
8
+
9
+ def initialize(worker)
10
+ @worker = worker
11
+ end
12
+
13
+ def work_loop
14
+ loop do
15
+ DispatcherQueue.notify
16
+ work
17
+ end
18
+ end
19
+
20
+ private
21
+
22
+ def rsync(logger, taskset)
23
+ logger.write("Start RSync")
24
+
25
+ working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
26
+ FileUtils.mkdir_p(working_path) unless Dir.exists?(working_path)
27
+ remote_path = "#{RSyncInfo.rsync_server}:#{File.join(RSyncInfo.rsync_dir, taskset.rsync_name)}"
28
+ command = "rsync #{RSyncInfo.rsync_options} #{remote_path}/ #{working_path}"
29
+
30
+ pid, out_rd, err_rd = execute_with_logs(working_path, command, {})
31
+ log_to_logger(logger, out_rd, err_rd)
32
+ pid, status = Process.waitpid2(pid)
33
+ if status.success?
34
+ logger.write("RSync finished")
35
+ return true
36
+ else
37
+ logger.write("RSync failed")
38
+ ArbiterQueue.fail(taskset)
39
+ return false
40
+ end
41
+ end
42
+
43
+ def setup(logger, taskset)
44
+ logger.write("Start setup")
45
+ env = {
46
+ 'NUM_SLAVES' => RRRSpec.configuration.slave_processes.to_s
47
+ }
48
+
49
+ working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
50
+ pid, out_rd, err_rd = execute_with_logs(working_path, '/bin/bash -ex', env,
51
+ taskset.setup_command)
52
+ log_to_logger(logger, out_rd, err_rd)
53
+ pid, status = Process.waitpid2(pid)
54
+ if status.success?
55
+ logger.write("Setup finished")
56
+ return true
57
+ else
58
+ logger.write("Setup failed")
59
+ ArbiterQueue.fail(taskset)
60
+ return false
61
+ end
62
+ end
63
+
64
+ def rspec(taskset)
65
+ working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
66
+
67
+ num_slaves = RRRSpec.configuration.slave_processes
68
+ env = {}
69
+ env["NUM_SLAVES"] = num_slaves.to_s
70
+ env["RRRSPEC_CONFIG_FILES"] = RRRSpec.configuration.loaded.join(':')
71
+ env["RRRSPEC_WORKING_DIR"] = RRRSpec.configuration.working_dir
72
+ env["RRRSPEC_TASKSET_KEY"] = taskset.key
73
+
74
+ pid_to_slave_number = {}
75
+ slave_command = taskset.slave_command
76
+ spawner = proc do |slave_number|
77
+ pid, out_rd, err_rd = execute_with_logs(
78
+ working_path, '/bin/bash -ex',
79
+ env.merge({"SLAVE_NUMBER" => slave_number.to_s}),
80
+ slave_command
81
+ )
82
+ slave = Slave.build_from_pid(pid)
83
+ taskset.add_slave(slave)
84
+ Thread.fork { log_to_logger(TimedLogger.new(slave), out_rd, err_rd) }
85
+
86
+ pid_to_slave_number[pid] = slave_number
87
+ end
88
+
89
+ num_slaves.times { |i| spawner.call(i) }
90
+
91
+ cancel_watcher_pid = Process.fork do
92
+ $0 = 'rrrspec cancel watcher'
93
+ loop do
94
+ break unless taskset.status == 'running'
95
+ sleep CANCEL_POLLING
96
+ end
97
+ end
98
+
99
+ trials = 1
100
+ max_trials = taskset.max_trials
101
+ loop do
102
+ break if pid_to_slave_number.empty?
103
+ begin
104
+ pid, status = Process.wait2
105
+ break if pid == cancel_watcher_pid
106
+ break unless taskset.status == 'running'
107
+
108
+ slave = Slave.build_from_pid(pid)
109
+ if status.success?
110
+ slave.update_status('normal_exit')
111
+ pid_to_slave_number.delete(pid)
112
+ else
113
+ exit_code = (status.to_i >> 8)
114
+ if exit_code == TIMEOUT_EXITCODE
115
+ slave_log = slave.log
116
+ slave.trials.each do |trial|
117
+ if trial.status == nil
118
+ trial.finish('timeout', slave_log, '', nil, nil, nil)
119
+ ArbiterQueue.trial(trial)
120
+ end
121
+ end
122
+ slave.update_status('timeout_exit')
123
+ else
124
+ slave.trials.each do |trial|
125
+ if trial.status == nil
126
+ trial.finish('error', '', '', nil, nil, nil)
127
+ ArbiterQueue.trial(trial)
128
+ end
129
+ end
130
+ slave.update_status('failure_exit')
131
+ trials += 1
132
+ if trials > max_trials
133
+ ArbiterQueue.fail(taskset)
134
+ break
135
+ end
136
+ end
137
+ slave_number = pid_to_slave_number[pid]
138
+ pid_to_slave_number.delete(pid)
139
+ spawner.call(slave_number)
140
+ end
141
+ rescue Errno::ECHILD
142
+ break
143
+ end
144
+ end
145
+ return cancel_watcher_pid, pid_to_slave_number
146
+ end
147
+
148
+ def cleaning_process(logger, taskset, cancel_watcher_pid, pid_to_slave_number)
149
+ logger.write("Send TERM signal to the children")
150
+ (pid_to_slave_number.keys + [cancel_watcher_pid]).each do |pid|
151
+ begin
152
+ Process.kill("-TERM", pid)
153
+ rescue Errno::ESRCH
154
+ end
155
+ end
156
+
157
+ logger.write("Wait for the children")
158
+ begin
159
+ loop do
160
+ pid, status = Process.wait2
161
+ if pid != cancel_watcher_pid
162
+ slave = Slave.build_from_pid(pid)
163
+ slave.update_status('normal_exit')
164
+ end
165
+ end
166
+ rescue Errno::ECHILD
167
+ end
168
+ logger.write("Finished the task")
169
+
170
+ # Some slaves are failed to exit with SIGTERM. Kill -9 them by name.
171
+ `ps aux | grep "rrrspec slave" | grep -v grep | awk '{print $2}'`.split("\n").map(&:to_i).each do |pid|
172
+ begin
173
+ Process.kill("KILL", pid)
174
+ rescue Errno::ESRCH
175
+ end
176
+ end
177
+ end
178
+
179
+ def work
180
+ @worker.update_current_taskset(nil)
181
+ taskset = @worker.dequeue_taskset
182
+ worker_log = WorkerLog.create(@worker, taskset)
183
+ logger = TimedLogger.new(worker_log)
184
+
185
+ check = proc do
186
+ unless taskset.status == 'running'
187
+ logger.write("The taskset(#{taskset.key}) is not running but #{taskset.status}")
188
+ return
189
+ end
190
+ end
191
+ check.call
192
+ @worker.update_current_taskset(taskset)
193
+
194
+ rsync(logger, taskset)
195
+ worker_log.set_rsync_finished_time
196
+ check.call
197
+
198
+ setup(logger, taskset)
199
+ worker_log.set_setup_finished_time
200
+ check.call
201
+
202
+ cancel_watcher_pid, pid_to_slave_number = rspec(taskset)
203
+ cleaning_process(logger, taskset, cancel_watcher_pid, pid_to_slave_number)
204
+ ensure
205
+ worker_log.set_finished_time if worker_log
206
+ @worker.update_current_taskset(nil)
207
+ end
208
+
209
+ def execute_with_logs(chdir, command, env, input=nil)
210
+ Bundler.with_clean_env do
211
+ in_rd, in_wt = IO.pipe
212
+ out_rd, out_wt = IO.pipe
213
+ err_rd, err_wt = IO.pipe
214
+ pid = spawn(env, command, { chdir: chdir, pgroup: true,
215
+ in: in_rd, out: out_wt, err: err_wt })
216
+ out_wt.close_write
217
+ err_wt.close_write
218
+ in_wt.write(input) if input
219
+ in_wt.close_write
220
+
221
+ return pid, out_rd, err_rd
222
+ end
223
+ end
224
+
225
+ def log_to_logger(logger, out_rd, err_rd)
226
+ rds = [out_rd, err_rd]
227
+ while !rds.empty?
228
+ IO.select(rds)[0].each do |r|
229
+ line = r.gets
230
+ if line
231
+ line = line.strip
232
+ if r == out_rd
233
+ logger.write("OUT " + line)
234
+ else
235
+ logger.write("ERR " + line)
236
+ end
237
+ else
238
+ rds.delete(r)
239
+ end
240
+ end
241
+ end
242
+ end
243
+ end
244
+ end
245
+ end
@@ -0,0 +1,9 @@
1
+ require 'active_record'
2
+
3
+ require 'rrrspec'
4
+ require 'rrrspec/server/arbiter'
5
+ require 'rrrspec/server/configuration'
6
+ require 'rrrspec/server/dispatcher'
7
+ require 'rrrspec/server/persistent_models'
8
+ require 'rrrspec/server/persister'
9
+ require 'rrrspec/server/worker_runner'
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'rrrspec/server/version'
5
+ require 'pathname'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = "rrrspec-server"
9
+ spec.version = RRRSpec::Server::VERSION
10
+ spec.authors = ["Masaya Suzuki"]
11
+ spec.email = ["draftcode@gmail.com"]
12
+ spec.description = "Execute RSpec in a distributed manner"
13
+ spec.summary = "Execute RSpec in a distributed manner"
14
+ spec.homepage = ""
15
+ spec.license = "MIT"
16
+
17
+ gemspec_dir = File.expand_path('..', __FILE__)
18
+ spec.files = `git ls-files`.split($/).
19
+ map { |f| File.absolute_path(f) }.
20
+ select { |f| f.start_with?(gemspec_dir) }.
21
+ map { |f| Pathname(f).relative_path_from(Pathname(gemspec_dir)).to_s }
22
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
24
+ spec.require_paths = ["lib"]
25
+
26
+ spec.add_development_dependency "database_cleaner", "~> 1.2.0"
27
+ spec.add_development_dependency "mysql2"
28
+ spec.add_development_dependency "rake"
29
+ spec.add_development_dependency "rspec"
30
+ spec.add_development_dependency "sqlite3"
31
+ spec.add_development_dependency "timecop"
32
+ spec.add_dependency "activerecord", "~> 3.0"
33
+ spec.add_dependency "activerecord-import", "~> 0.3.1"
34
+ spec.add_dependency "activesupport"
35
+ spec.add_dependency "bundler"
36
+ spec.add_dependency "redis"
37
+ spec.add_dependency "rrrspec-client"
38
+ spec.add_dependency "thor"
39
+ end
data/spec/fixture.rb ADDED
@@ -0,0 +1,32 @@
1
+ module RRRSpec
2
+ def self.finished_fullset
3
+ worker = Worker.create('default')
4
+ taskset = Taskset.create(
5
+ 'testuser', 'echo 1', 'echo 2', 'default', 'default', 3, 3, 5, 5
6
+ )
7
+ task = Task.create(taskset, 10, 'spec/test_spec.rb')
8
+ taskset.add_task(task)
9
+ taskset.enqueue_task(task)
10
+ ActiveTaskset.add(taskset)
11
+ worker_log = WorkerLog.create(worker, taskset)
12
+ worker_log.set_rsync_finished_time
13
+ worker_log.append_log('worker_log log body')
14
+ worker_log.set_setup_finished_time
15
+ slave = Slave.create
16
+ taskset.add_slave(slave)
17
+ slave.append_log('slave log body')
18
+ trial = Trial.create(task, slave)
19
+ trial.start
20
+ trial.finish('pending', 'stdout body', 'stderr body', 10, 2, 0)
21
+ task.update_status('pending')
22
+ taskset.incr_succeeded_count
23
+ taskset.finish_task(task)
24
+ taskset.update_status('succeeded')
25
+ taskset.set_finished_time
26
+ ActiveTaskset.remove(taskset)
27
+ slave.update_status('normal_exit')
28
+ worker_log.set_finished_time
29
+
30
+ return worker, taskset, task, worker_log, slave, trial
31
+ end
32
+ end