rrrspec-server 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ffdf010821a66bdb732d362cfd9d3c33acf9e6ad
+  data.tar.gz: 06a8ab51406619a53ffa3131c8a356d630b6e2ec
+SHA512:
+  metadata.gz: 3b86f71da951714df9558c4918e6bcf36b29815d97aeb907e71212762566f089baaf6e6fcf514df6e3569c8bdec17b11e50690c0df3d803eb4079805132c45d1
+  data.tar.gz: 3c0d6cd34faa4fc5b2853775c890053e27761f5d7e1f9eed510323b487e33206c82b983ce93b07d53e7ab8a1b5c4bf1b12565ea0f45adb29f51b151e72d8076d

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'rrrspec-client', path: '../'
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+load File.expand_path('../tasks/db.rake', __FILE__)

data/bin/rrrspec-server

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'rrrspec/server/cli'
+RRRSpec::Server::CLI.start

data/db/migrate/20131105050718_create_tables.rb

@@ -0,0 +1,77 @@
+class CreateTables < ActiveRecord::Migration
+  def change
+    create_table(:tasksets) do |t|
+      t.string :key
+      t.string :rsync_name
+      t.text :setup_command
+      t.text :slave_command
+      t.string :worker_type
+      t.integer :max_workers
+      t.integer :max_trials
+      t.string :taskset_class
+      t.integer :unknown_spec_timeout_sec
+      t.integer :least_timeout_sec
+      t.datetime :created_at
+
+      t.string :status
+      t.datetime :finished_at
+      t.text :log
+    end
+    add_index :tasksets, :key
+    add_index :tasksets, :rsync_name
+    add_index :tasksets, :taskset_class
+    add_index :tasksets, :created_at
+    add_index :tasksets, :status
+
+    create_table(:tasks) do |t|
+      t.string :key
+      t.references :taskset
+      t.string :status
+      t.integer :estimate_sec
+      t.string :spec_file
+    end
+    add_index :tasks, :key
+    add_index :tasks, :taskset_id
+    add_index :tasks, :status
+
+    create_table(:trials) do |t|
+      t.string :key
+      t.references :task
+      t.references :slave
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.string :status
+      t.text :stdout
+      t.text :stderr
+      t.integer :passed
+      t.integer :pending
+      t.integer :failed
+    end
+    add_index :trials, :key
+    add_index :trials, :task_id
+    add_index :trials, :slave_id
+
+    create_table(:worker_logs) do |t|
+      t.string :key
+      t.string :worker_key
+      t.references :taskset
+      t.datetime :started_at
+      t.datetime :rsync_finished_at
+      t.datetime :setup_finished_at
+      t.datetime :finished_at
+      t.text :log
+    end
+    add_index :worker_logs, :key
+    add_index :worker_logs, :worker_key
+    add_index :worker_logs, :taskset_id
+
+    create_table(:slaves) do |t|
+      t.string :key
+      t.references :taskset
+      t.string :status
+      t.text :log
+    end
+    add_index :slaves, :key
+    add_index :slaves, :taskset_id
+  end
+end

data/db/schema.rb

@@ -0,0 +1,94 @@
+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 20131105050718) do
+
+  create_table "slaves", :force => true do |t|
+    t.string  "key"
+    t.integer "taskset_id"
+    t.string  "status"
+    t.text    "log"
+  end
+
+  add_index "slaves", ["key"], :name => "index_slaves_on_key"
+  add_index "slaves", ["taskset_id"], :name => "index_slaves_on_taskset_id"
+
+  create_table "tasks", :force => true do |t|
+    t.string  "key"
+    t.integer "taskset_id"
+    t.string  "status"
+    t.integer "estimate_sec"
+    t.string  "spec_file"
+  end
+
+  add_index "tasks", ["key"], :name => "index_tasks_on_key"
+  add_index "tasks", ["status"], :name => "index_tasks_on_status"
+  add_index "tasks", ["taskset_id"], :name => "index_tasks_on_taskset_id"
+
+  create_table "tasksets", :force => true do |t|
+    t.string   "key"
+    t.string   "rsync_name"
+    t.text     "setup_command"
+    t.text     "slave_command"
+    t.string   "worker_type"
+    t.integer  "max_workers"
+    t.integer  "max_trials"
+    t.string   "taskset_class"
+    t.integer  "unknown_spec_timeout_sec"
+    t.integer  "least_timeout_sec"
+    t.datetime "created_at"
+    t.string   "status"
+    t.datetime "finished_at"
+    t.text     "log"
+  end
+
+  add_index "tasksets", ["created_at"], :name => "index_tasksets_on_created_at"
+  add_index "tasksets", ["key"], :name => "index_tasksets_on_key"
+  add_index "tasksets", ["rsync_name"], :name => "index_tasksets_on_rsync_name"
+  add_index "tasksets", ["status"], :name => "index_tasksets_on_status"
+  add_index "tasksets", ["taskset_class"], :name => "index_tasksets_on_taskset_class"
+
+  create_table "trials", :force => true do |t|
+    t.string   "key"
+    t.integer  "task_id"
+    t.integer  "slave_id"
+    t.datetime "started_at"
+    t.datetime "finished_at"
+    t.string   "status"
+    t.text     "stdout"
+    t.text     "stderr"
+    t.integer  "passed"
+    t.integer  "pending"
+    t.integer  "failed"
+  end
+
+  add_index "trials", ["key"], :name => "index_trials_on_key"
+  add_index "trials", ["slave_id"], :name => "index_trials_on_slave_id"
+  add_index "trials", ["task_id"], :name => "index_trials_on_task_id"
+
+  create_table "worker_logs", :force => true do |t|
+    t.string   "key"
+    t.string   "worker_key"
+    t.integer  "taskset_id"
+    t.datetime "started_at"
+    t.datetime "rsync_finished_at"
+    t.datetime "setup_finished_at"
+    t.datetime "finished_at"
+    t.text     "log"
+  end
+
+  add_index "worker_logs", ["key"], :name => "index_worker_logs_on_key"
+  add_index "worker_logs", ["taskset_id"], :name => "index_worker_logs_on_taskset_id"
+  add_index "worker_logs", ["worker_key"], :name => "index_worker_logs_on_worker_key"
+
+end

data/lib/rrrspec/server/arbiter.rb

@@ -0,0 +1,186 @@
+module RRRSpec
+  module Server
+    module Arbiter
+      module_function
+
+      def work_loop
+        loop { work }
+      end
+
+      def work
+        command, arg = ArbiterQueue.dequeue
+        case command
+        when 'cancel'
+          cancel(arg)
+        when 'check'
+          check(arg)
+        when 'fail'
+          fail(arg)
+        when 'trial'
+          trial(arg)
+        end
+      end
+
+      private
+      module_function
+
+      def cancel(taskset)
+        logger = TimedLogger.new(taskset)
+        logger.write("Cancel requested")
+
+        if [nil, 'running'].include?(taskset.status)
+          taskset.update_status('cancelled')
+          taskset.set_finished_time
+          taskset.clear_queue
+        end
+        PersisterQueue.enqueue(taskset)
+        ActiveTaskset.remove(taskset)
+      end
+
+      def check(taskset)
+        logger = TimedLogger.new(taskset)
+        logger.write("Check requested")
+
+        unless taskset.status == 'running'
+          PersisterQueue.enqueue(taskset)
+          ActiveTaskset.remove(taskset)
+          return
+        end
+
+        tasks_left = taskset.tasks_left
+        tasks_left.each do |task|
+          check_task(logger, taskset, task)
+        end
+
+        tasks_left = taskset.tasks_left
+        if tasks_left.empty?
+          if taskset.failed_count == 0
+            taskset.update_status('succeeded')
+          else
+            taskset.update_status('failed')
+          end
+          taskset.set_finished_time
+          PersisterQueue.enqueue(taskset)
+          ActiveTaskset.remove(taskset)
+        elsif taskset.queue_empty?
+          requeue_speculative(logger, taskset, tasks_left)
+        end
+      end
+
+      def check_task(logger, taskset, task)
+        running_trial = []
+        passed_trial = []
+        pending_trial = []
+        failed_trial = []
+        trials = task.trials
+        trials.each do |trial|
+          case trial.status
+          when nil, ''
+            if trial.slave.exist?
+              running_trial << trial
+            else
+              trial.finish('error', '', 'Failed by Arbiter', nil, nil, nil)
+              failed_trial << trial
+            end
+          when 'passed'
+            passed_trial << trial
+          when 'pending'
+            pending_trial << trial
+          when 'failed', 'error', 'timeout'
+            failed_trial << trial
+          end
+        end
+        case
+        when !passed_trial.empty?
+          task.update_status('passed')
+          taskset.incr_succeeded_count
+          taskset.finish_task(task)
+        when !pending_trial.empty?
+          task.update_status('pending')
+          taskset.incr_succeeded_count
+          taskset.finish_task(task)
+        when !running_trial.empty?
+          # Do nothing
+        when failed_trial.size >= taskset.max_trials
+          logger.write("Mark failed #{task.key}")
+          task.update_status('failed')
+          taskset.incr_failed_count
+          taskset.finish_task(task)
+        end
+      end
+
+      def requeue_speculative(logger, taskset, tasks_left)
+        running_tasks = Hash.new { |hash,key| hash[key] = [] }
+        not_running_tasks = []
+        tasks_left.each do |task|
+          trials = task.trials
+          if trials.any? { |trial| trial.status.blank? }
+            running_tasks[trials.size] << task
+          else
+            not_running_tasks << task
+          end
+        end
+
+        if not_running_tasks.empty?
+          task = running_tasks[running_tasks.keys.min].sample
+          logger.write("Speculatively enqueue the task #{task.key}")
+          taskset.enqueue_task(task)
+        else
+          task = not_running_tasks.sample
+          logger.write("Enqueue the task #{task.key}")
+          taskset.enqueue_task(task)
+        end
+      end
+
+      def fail(taskset)
+        logger = TimedLogger.new(taskset)
+        logger.write("Fail requested")
+
+        if [nil, 'running'].include?(taskset.status)
+          taskset.update_status('failed')
+          taskset.set_finished_time
+          taskset.clear_queue
+        end
+        PersisterQueue.enqueue(taskset)
+        ActiveTaskset.remove(taskset)
+      end
+
+      def trial(trial)
+        task = trial.task
+        return if task.status.present?
+        taskset = task.taskset
+        logger = TimedLogger.new(taskset)
+
+        if trial.status == nil
+          trial.finish('error', '', 'Failed by Arbiter', nil, nil, nil)
+        end
+
+        case trial.status
+        when 'passed'
+          task.update_status('passed')
+          taskset.incr_succeeded_count
+          taskset.finish_task(task)
+        when 'pending'
+          task.update_status('pending')
+          taskset.incr_succeeded_count
+          taskset.finish_task(task)
+        when 'failed', 'error', 'timeout'
+          trials = task.trials
+          finished_trials = []
+          trials.each do |trial|
+            finished_trials << trial unless [nil, ''].include?(trial.status)
+          end
+          if finished_trials.size >= taskset.max_trials
+            task.update_status('failed')
+            taskset.incr_failed_count
+            taskset.finish_task(task)
+          elsif trials.size < taskset.max_trials
+            task.update_status(nil)
+            logger.write("Enqueue the task #{task.key}")
+            taskset.reversed_enqueue_task(task)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/cli.rb

@@ -0,0 +1,107 @@
+require 'rrrspec/server'
+require 'thor'
+
+module RRRSpec
+  module Server
+    class CLI < Thor
+      package_name 'RRRSpec'
+      default_command 'help'
+      class_option :config, aliases: '-c',  type: :string, default: ''
+
+      no_commands do
+        def setup(conf)
+          RRRSpec.setup(conf, options[:config].split(':'))
+        end
+
+        def log_exception
+          yield
+        rescue
+          RRRSpec.logger.error($!)
+          raise
+        end
+
+        def auto_rebirth
+          Signal.trap('TERM') do
+            Signal.trap('TERM', 'DEFAULT')
+            Process.kill('TERM', 0)
+          end
+          loop do
+            pid = Process.fork do
+              log_exception { yield }
+            end
+            Process.waitpid(pid)
+          end
+        end
+
+        def daemonize
+          return unless options[:daemonize]
+          pidfile = options[:pidfile]
+
+          if pidfile
+            pidfile = File.absolute_path(pidfile)
+            if File.exists?(pidfile)
+              pid = open(pidfile, 'r') { |f| f.read.strip }
+              if pid
+                begin
+                  if Process.kill(0, pid.to_i) == 1
+                    $stderr.puts "Daemon process already exists: #{pid}"
+                    exit 1
+                  end
+                rescue Errno::ESRCH
+                end
+              end
+            end
+          end
+
+          Process.daemon
+
+          if pidfile
+            open(pidfile, 'w') { |f| f.write(Process.pid.to_s) }
+
+            parent_pid = Process.pid
+            at_exit do
+              if Process.pid == parent_pid
+                File.delete(pidfile)
+              end
+            end
+          end
+        end
+      end
+
+      method_option :daemonize, type: :boolean
+      method_option :pidfile, type: :string
+      desc 'server', 'Run RRRSpec as a server'
+      def server
+        $0 = 'rrrspec server'
+        setup(ServerConfiguration.new)
+        daemonize
+        auto_rebirth do
+          ActiveRecord::Base.establish_connection(**RRRSpec.configuration.persistence_db)
+          Thread.abort_on_exception = true
+          Thread.fork { RRRSpec.pacemaker(RSyncInfo, 60, 5) }
+          Thread.fork { Dispatcher.work_loop }
+          Thread.fork { Arbiter.work_loop }
+          Thread.fork { Persister.work_loop }
+          Kernel.sleep
+        end
+      end
+
+      method_option :daemonize, type: :boolean
+      method_option :pidfile, type: :string
+      desc 'worker', 'Run RRRSpec as a worker'
+      def worker
+        $0 = 'rrrspec worker'
+        setup(WorkerConfiguration.new)
+        daemonize
+        auto_rebirth do
+          worker = Worker.create(RRRSpec.configuration.worker_type)
+          worker_runner = WorkerRunner.new(worker)
+          Thread.abort_on_exception = true
+          Thread.fork { RRRSpec.pacemaker(worker, 60, 5) }
+          Thread.fork { worker_runner.work_loop }
+          Kernel.sleep
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/configuration.rb

@@ -0,0 +1,54 @@
+require 'facter'
+
+module RRRSpec
+  module Server
+    class ServerConfiguration < Configuration
+      attr_accessor :rsync_server, :rsync_dir, :rsync_options
+      attr_accessor :persistence_db
+      attr_accessor :json_cache_path
+
+      def initialize
+        super()
+        @type = :server
+      end
+
+      def check_validity
+        validity = super
+
+        unless rsync_server and rsync_options and rsync_dir
+          $stderr.puts('The rsync options are not set')
+          validity = false
+        end
+
+        unless persistence_db
+          $stderr.puts('The database options are not set')
+          validity = false
+        end
+
+        validity
+      end
+    end
+
+    class WorkerConfiguration < Configuration
+      attr_accessor :working_dir, :worker_type, :slave_processes
+
+      def initialize
+        super()
+        @slave_processes = Facter.processorcount.to_i
+        @worker_type = 'default'
+        @type = :worker
+      end
+
+      def check_validity
+        validity = super
+
+        unless working_dir and worker_type
+          $stderr.puts('The worker options are not set')
+          validity = false
+        end
+
+        validity
+      end
+    end
+  end
+end

data/lib/rrrspec/server/dispatcher.rb

@@ -0,0 +1,53 @@
+module RRRSpec
+  module Server
+    class Dispatcher
+      def self.work
+        assigned = Hash.new { |hash,key| hash[key] = [] }
+        unemployed = Hash.new { |hash,key| hash[key] = [] }
+        Worker.list.each do |worker|
+          unless worker.exist?
+            worker.evict
+          else
+            taskset = worker.current_taskset
+            if taskset
+              assigned[taskset.key] << worker
+            else
+              unemployed[worker.worker_type] << worker
+            end
+          end
+        end
+
+        ActiveTaskset.list.each do |taskset|
+          should_mark_running = false
+          case taskset.status
+          when 'succeeded', 'cancelled', 'failed'
+            next
+          when nil
+            should_mark_running = true
+          end
+
+          # Cache the values
+          max_workers = taskset.max_workers
+          worker_type = taskset.worker_type
+          while max_workers > assigned[taskset.key].size
+            break if unemployed[worker_type].empty?
+            worker = unemployed[worker_type].pop
+            if should_mark_running
+              taskset.update_status('running')
+              should_mark_running = false
+            end
+            worker.enqueue_taskset(taskset)
+            assigned[taskset.key] << worker
+          end
+        end
+      end
+
+      def self.work_loop
+        loop do
+          DispatcherQueue.wait
+          work
+        end
+      end
+    end
+  end
+end