rrrspec-server 0.2.0

data/lib/rrrspec/server/persistent_models.rb

@@ -0,0 +1,111 @@
+module RRRSpec
+  module Server
+    module Persistence
+      class Taskset < ActiveRecord::Base
+        include ActiveModel::Serializers::JSON
+
+        has_many :worker_logs
+        has_many :slaves
+        has_many :tasks
+
+        scope :full, includes(
+          :tasks => [{:trials => [:task, :slave]}, :taskset],
+          :slaves => [:trials],
+          :worker_logs => [:taskset]
+        )
+
+        def as_nodetail_json
+          as_json(except: :id)
+        end
+
+        def as_short_json
+          h = as_json(except: :id)
+          h['slaves'] = slaves.map { |slave| slave.as_json(only: :key) }
+          h['tasks'] = tasks.map { |task| task.as_json(only: :key) }
+          h['worker_logs'] = worker_logs.map { |worker_log| worker_log.as_json(only: :key) }
+          h
+        end
+
+        def as_full_json
+          h = as_json(except: :id)
+          h['slaves'] = slaves.map(&:as_full_json)
+          h['tasks'] = tasks.map(&:as_full_json)
+          h['worker_logs'] = worker_logs.map(&:as_full_json)
+          h
+        end
+      end
+
+      class Task < ActiveRecord::Base
+        include ActiveModel::Serializers::JSON
+
+        belongs_to :taskset
+        has_many :trials
+
+        def as_short_json
+          h = as_json(except: [:id, :taskset_id, :trials],
+                      include: { 'taskset' => { only: :key } })
+          h['trials'] = trials.map { |trial| trial.as_json(only: :key) }
+          h
+        end
+
+        def as_full_json
+          h = as_json(except: [:id, :taskset_id, :trials],
+                      include: { 'taskset' => { only: :key } })
+          h['trials'] = trials.map(&:as_full_json)
+          h
+        end
+      end
+
+      class Trial < ActiveRecord::Base
+        include ActiveModel::Serializers::JSON
+
+        belongs_to :task
+        belongs_to :slave
+
+        def as_full_json
+          as_json(except: [:id, :task_id, :slave_id],
+                  include: { 'slave' => { only: :key }, 'task' => { only: :key } })
+        end
+
+        def as_short_json
+          as_full_json
+        end
+      end
+
+      class WorkerLog < ActiveRecord::Base
+        include ActiveModel::Serializers::JSON
+
+        belongs_to :taskset
+
+        def as_full_json
+          as_json(except: [:id, :taskset_id, :worker_key],
+                  include: { 'taskset' => { only: :key } },
+                  methods: :worker)
+        end
+
+        def as_short_json
+          as_full_json
+        end
+
+        def worker
+          { 'key' => worker_key }
+        end
+      end
+
+      class Slave < ActiveRecord::Base
+        include ActiveModel::Serializers::JSON
+
+        has_many :trials
+
+        def as_full_json
+          as_json(except: [:id, :taskset_id],
+                  include: { 'trials' => { only: :key } })
+        end
+
+        def as_short_json
+          as_full_json
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/persister.rb

@@ -0,0 +1,147 @@
+require 'zlib'
+require 'activerecord-import'
+require 'active_support/inflector'
+ActiveSupport::Inflector::Inflections.instance.singular('Slaves', 'Slave')
+ActiveSupport::Inflector::Inflections.instance.singular('slaves', 'slave')
+ActiveRecord::Base.include_root_in_json = false
+ActiveRecord::Base.default_timezone = :utc
+
+module RRRSpec
+  module Server
+    module Persister
+      SLAVE_EXIT_WAIT_TIME = 15
+      PERSISTED_RESIDUE_SEC = 60
+
+      module_function
+
+      def work_loop
+        loop { work }
+      end
+
+      def work
+        taskset = PersisterQueue.dequeue
+        ActiveRecord::Base.connection_pool.with_connection do
+          return if Persistence::Taskset.where(key: taskset.key).exists?
+        end
+
+        sleep SLAVE_EXIT_WAIT_TIME
+
+        ActiveRecord::Base.connection_pool.with_connection do
+          persist(taskset)
+          if RRRSpec.configuration.json_cache_path
+            create_api_cache(taskset, RRRSpec.configuration.json_cache_path)
+          end
+          taskset.expire(PERSISTED_RESIDUE_SEC)
+          update_estimate_sec(taskset)
+        end
+      rescue
+        RRRSpec.logger.error($!)
+      end
+
+      private
+      module_function
+
+      def persist(taskset)
+        taskset_finished_at = taskset.finished_at
+        return if taskset_finished_at.blank?
+
+        p_taskset = ActiveRecord::Base.transaction do
+          h = taskset.to_h
+          h.delete('tasks')
+          h.delete('slaves')
+          h.delete('worker_logs')
+          Persistence::Taskset.create(h)
+        end
+
+        ActiveRecord::Base.transaction do
+          p_slaves = taskset.slaves.map do |slave|
+            h = slave.to_h
+            h.delete('trials')
+            p_slave = Persistence::Slave.new(h)
+            p_slave.taskset_id = p_taskset.id
+            p_slave
+          end
+          Persistence::Slave.import(p_slaves)
+        end
+
+        ActiveRecord::Base.transaction do
+          Persistence::Task.import(taskset.tasks.map do |task|
+            h = task.to_h
+            h.delete('taskset')
+            h.delete('trials')
+            p_task = Persistence::Task.new(h)
+            p_task.taskset_id = p_taskset
+            p_task
+          end)
+        end
+
+        p_slaves = {}
+        p_taskset.slaves.each do |p_slave|
+          p_slaves[p_slave.key] = p_slave
+        end
+
+        ActiveRecord::Base.transaction do
+          p_trials = []
+          p_taskset.tasks.each do |p_task|
+            Task.new(p_task.key).trials.each do |trial|
+              h = trial.to_h
+              next if h['finished_at'].blank? || h['finished_at'] > taskset_finished_at
+              slave_key = h.delete('slave')['key']
+              h.delete('task')
+              p_trial = Persistence::Trial.new(h)
+              p_trial.task_id = p_task
+              p_trial.slave_id = p_slaves[slave_key]
+              p_trials << p_trial
+            end
+          end
+          Persistence::Trial.import(p_trials)
+        end
+
+        ActiveRecord::Base.transaction do
+          Persistence::WorkerLog.import(taskset.worker_logs.map do |worker_log|
+            h = worker_log.to_h
+            h['worker_key'] = h['worker']['key']
+            h.delete('worker')
+            h.delete('taskset')
+            p_worker_log = Persistence::WorkerLog.new(h)
+            p_worker_log.taskset_id = p_taskset
+            p_worker_log
+          end)
+        end
+      end
+
+      def create_api_cache(taskset, path)
+        p_obj = Persistence::Taskset.where(key: taskset.key).full.first
+        json = JSON.generate(p_obj.as_full_json.update('is_full' => true))
+
+        FileUtils.mkdir_p(File.join(path, 'v1', 'tasksets'))
+        json_path = File.join(path, 'v1', 'tasksets', taskset.key.gsub(':', '-'))
+        IO.write(json_path, json)
+        Zlib::GzipWriter.open(json_path + ".gz") { |gz| gz.write(json) }
+      end
+
+      ESTIMATION_FIELDS = [
+        "`spec_file`",
+        "avg(UNIX_TIMESTAMP(`trials`.`finished_at`)-UNIX_TIMESTAMP(`trials`.`started_at`)) as `avg`",
+        # "avg(`trials`.`finished_at`-`trials`.`started_at`) as `avg`",
+      ]
+
+      def update_estimate_sec(taskset)
+        p_obj = Persistence::Taskset.where(key: taskset.key).first
+        taskset_class = p_obj.taskset_class
+        query = Persistence::Task.joins(:trials).joins(:taskset).
+          select(ESTIMATION_FIELDS).
+          where('tasksets.taskset_class' => taskset_class).
+          where('trials.status' => ["passed", "pending"]).
+          group('spec_file')
+        estimation = {}
+        query.each do |row|
+          estimation[row.spec_file] = row.avg.to_i
+        end
+        unless estimation.empty?
+          TasksetEstimation.update_estimate_secs(taskset_class, estimation)
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/version.rb

@@ -0,0 +1,5 @@
+module RRRSpec
+  module Server
+    VERSION = "0.2.0"
+  end
+end

data/lib/rrrspec/server/worker_runner.rb

@@ -0,0 +1,245 @@
+module RRRSpec
+  module Server
+    class WorkerRunner
+      CANCEL_POLLING = 10
+      TIMEOUT_EXITCODE = 42
+
+      attr_reader :internal_status, :current_taskset
+
+      def initialize(worker)
+        @worker = worker
+      end
+
+      def work_loop
+        loop do
+          DispatcherQueue.notify
+          work
+        end
+      end
+
+      private
+
+      def rsync(logger, taskset)
+        logger.write("Start RSync")
+
+        working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
+        FileUtils.mkdir_p(working_path) unless Dir.exists?(working_path)
+        remote_path = "#{RSyncInfo.rsync_server}:#{File.join(RSyncInfo.rsync_dir, taskset.rsync_name)}"
+        command = "rsync #{RSyncInfo.rsync_options} #{remote_path}/ #{working_path}"
+
+        pid, out_rd, err_rd = execute_with_logs(working_path, command, {})
+        log_to_logger(logger, out_rd, err_rd)
+        pid, status = Process.waitpid2(pid)
+        if status.success?
+          logger.write("RSync finished")
+          return true
+        else
+          logger.write("RSync failed")
+          ArbiterQueue.fail(taskset)
+          return false
+        end
+      end
+
+      def setup(logger, taskset)
+        logger.write("Start setup")
+        env = {
+          'NUM_SLAVES' => RRRSpec.configuration.slave_processes.to_s
+        }
+
+        working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
+        pid, out_rd, err_rd = execute_with_logs(working_path, '/bin/bash -ex', env,
+                                                taskset.setup_command)
+        log_to_logger(logger, out_rd, err_rd)
+        pid, status = Process.waitpid2(pid)
+        if status.success?
+          logger.write("Setup finished")
+          return true
+        else
+          logger.write("Setup failed")
+          ArbiterQueue.fail(taskset)
+          return false
+        end
+      end
+
+      def rspec(taskset)
+        working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
+
+        num_slaves = RRRSpec.configuration.slave_processes
+        env = {}
+        env["NUM_SLAVES"] = num_slaves.to_s
+        env["RRRSPEC_CONFIG_FILES"] = RRRSpec.configuration.loaded.join(':')
+        env["RRRSPEC_WORKING_DIR"] = RRRSpec.configuration.working_dir
+        env["RRRSPEC_TASKSET_KEY"] = taskset.key
+
+        pid_to_slave_number = {}
+        slave_command = taskset.slave_command
+        spawner = proc do |slave_number|
+          pid, out_rd, err_rd = execute_with_logs(
+            working_path, '/bin/bash -ex',
+            env.merge({"SLAVE_NUMBER" => slave_number.to_s}),
+            slave_command
+          )
+          slave = Slave.build_from_pid(pid)
+          taskset.add_slave(slave)
+          Thread.fork { log_to_logger(TimedLogger.new(slave), out_rd, err_rd) }
+
+          pid_to_slave_number[pid] = slave_number
+        end
+
+        num_slaves.times { |i| spawner.call(i) }
+
+        cancel_watcher_pid = Process.fork do
+          $0 = 'rrrspec cancel watcher'
+          loop do
+            break unless taskset.status == 'running'
+            sleep CANCEL_POLLING
+          end
+        end
+
+        trials = 1
+        max_trials = taskset.max_trials
+        loop do
+          break if pid_to_slave_number.empty?
+          begin
+            pid, status = Process.wait2
+            break if pid == cancel_watcher_pid
+            break unless taskset.status == 'running'
+
+            slave = Slave.build_from_pid(pid)
+            if status.success?
+              slave.update_status('normal_exit')
+              pid_to_slave_number.delete(pid)
+            else
+              exit_code = (status.to_i >> 8)
+              if exit_code == TIMEOUT_EXITCODE
+                slave_log = slave.log
+                slave.trials.each do |trial|
+                  if trial.status == nil
+                    trial.finish('timeout', slave_log, '', nil, nil, nil)
+                    ArbiterQueue.trial(trial)
+                  end
+                end
+                slave.update_status('timeout_exit')
+              else
+                slave.trials.each do |trial|
+                  if trial.status == nil
+                    trial.finish('error', '', '', nil, nil, nil)
+                    ArbiterQueue.trial(trial)
+                  end
+                end
+                slave.update_status('failure_exit')
+                trials += 1
+                if trials > max_trials
+                  ArbiterQueue.fail(taskset)
+                  break
+                end
+              end
+              slave_number = pid_to_slave_number[pid]
+              pid_to_slave_number.delete(pid)
+              spawner.call(slave_number)
+            end
+          rescue Errno::ECHILD
+            break
+          end
+        end
+        return cancel_watcher_pid, pid_to_slave_number
+      end
+
+      def cleaning_process(logger, taskset, cancel_watcher_pid, pid_to_slave_number)
+        logger.write("Send TERM signal to the children")
+        (pid_to_slave_number.keys + [cancel_watcher_pid]).each do |pid|
+          begin
+            Process.kill("-TERM", pid)
+          rescue Errno::ESRCH
+          end
+        end
+
+        logger.write("Wait for the children")
+        begin
+          loop do
+            pid, status = Process.wait2
+            if pid != cancel_watcher_pid
+              slave = Slave.build_from_pid(pid)
+              slave.update_status('normal_exit')
+            end
+          end
+        rescue Errno::ECHILD
+        end
+        logger.write("Finished the task")
+
+        # Some slaves are failed to exit with SIGTERM. Kill -9 them by name.
+        `ps aux | grep "rrrspec slave" | grep -v grep | awk '{print $2}'`.split("\n").map(&:to_i).each do |pid|
+          begin
+            Process.kill("KILL", pid)
+          rescue Errno::ESRCH
+          end
+        end
+      end
+
+      def work
+        @worker.update_current_taskset(nil)
+        taskset = @worker.dequeue_taskset
+        worker_log = WorkerLog.create(@worker, taskset)
+        logger = TimedLogger.new(worker_log)
+
+        check = proc do
+          unless taskset.status == 'running'
+            logger.write("The taskset(#{taskset.key}) is not running but #{taskset.status}")
+            return
+          end
+        end
+        check.call
+        @worker.update_current_taskset(taskset)
+
+        rsync(logger, taskset)
+        worker_log.set_rsync_finished_time
+        check.call
+
+        setup(logger, taskset)
+        worker_log.set_setup_finished_time
+        check.call
+
+        cancel_watcher_pid, pid_to_slave_number = rspec(taskset)
+        cleaning_process(logger, taskset, cancel_watcher_pid, pid_to_slave_number)
+      ensure
+        worker_log.set_finished_time if worker_log
+        @worker.update_current_taskset(nil)
+      end
+
+      def execute_with_logs(chdir, command, env, input=nil)
+        Bundler.with_clean_env do
+          in_rd, in_wt = IO.pipe
+          out_rd, out_wt = IO.pipe
+          err_rd, err_wt = IO.pipe
+          pid = spawn(env, command, { chdir: chdir, pgroup: true,
+                                      in: in_rd, out: out_wt, err: err_wt })
+          out_wt.close_write
+          err_wt.close_write
+          in_wt.write(input) if input
+          in_wt.close_write
+
+          return pid, out_rd, err_rd
+        end
+      end
+
+      def log_to_logger(logger, out_rd, err_rd)
+        rds = [out_rd, err_rd]
+        while !rds.empty?
+          IO.select(rds)[0].each do |r|
+            line = r.gets
+            if line
+              line = line.strip
+              if r == out_rd
+                logger.write("OUT " + line)
+              else
+                logger.write("ERR " + line)
+              end
+            else
+              rds.delete(r)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server.rb

@@ -0,0 +1,9 @@
+require 'active_record'
+
+require 'rrrspec'
+require 'rrrspec/server/arbiter'
+require 'rrrspec/server/configuration'
+require 'rrrspec/server/dispatcher'
+require 'rrrspec/server/persistent_models'
+require 'rrrspec/server/persister'
+require 'rrrspec/server/worker_runner'

data/rrrspec-server.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rrrspec/server/version'
+require 'pathname'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rrrspec-server"
+  spec.version       = RRRSpec::Server::VERSION
+  spec.authors       = ["Masaya Suzuki"]
+  spec.email         = ["draftcode@gmail.com"]
+  spec.description   = "Execute RSpec in a distributed manner"
+  spec.summary       = "Execute RSpec in a distributed manner"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  gemspec_dir = File.expand_path('..', __FILE__)
+  spec.files  = `git ls-files`.split($/).
+    map { |f| File.absolute_path(f) }.
+    select { |f| f.start_with?(gemspec_dir) }.
+    map { |f| Pathname(f).relative_path_from(Pathname(gemspec_dir)).to_s }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "database_cleaner", "~> 1.2.0"
+  spec.add_development_dependency "mysql2"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "timecop"
+  spec.add_dependency "activerecord", "~> 3.0"
+  spec.add_dependency "activerecord-import", "~> 0.3.1"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "bundler"
+  spec.add_dependency "redis"
+  spec.add_dependency "rrrspec-client"
+  spec.add_dependency "thor"
+end

data/spec/fixture.rb

@@ -0,0 +1,32 @@
+module RRRSpec
+  def self.finished_fullset
+    worker = Worker.create('default')
+    taskset = Taskset.create(
+      'testuser', 'echo 1', 'echo 2', 'default', 'default', 3, 3, 5, 5
+    )
+    task = Task.create(taskset, 10, 'spec/test_spec.rb')
+    taskset.add_task(task)
+    taskset.enqueue_task(task)
+    ActiveTaskset.add(taskset)
+    worker_log = WorkerLog.create(worker, taskset)
+    worker_log.set_rsync_finished_time
+    worker_log.append_log('worker_log log body')
+    worker_log.set_setup_finished_time
+    slave = Slave.create
+    taskset.add_slave(slave)
+    slave.append_log('slave log body')
+    trial = Trial.create(task, slave)
+    trial.start
+    trial.finish('pending', 'stdout body', 'stderr body', 10, 2, 0)
+    task.update_status('pending')
+    taskset.incr_succeeded_count
+    taskset.finish_task(task)
+    taskset.update_status('succeeded')
+    taskset.set_finished_time
+    ActiveTaskset.remove(taskset)
+    slave.update_status('normal_exit')
+    worker_log.set_finished_time
+
+    return worker, taskset, task, worker_log, slave, trial
+  end
+end