rrrspec-server 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffdf010821a66bdb732d362cfd9d3c33acf9e6ad
-  data.tar.gz: 06a8ab51406619a53ffa3131c8a356d630b6e2ec
+  metadata.gz: 3dd5351ecddcebbc6fbc7f89e1102c06b02dcb21
+  data.tar.gz: 33b7df062b10d6f6d9099372c931d85663bb68bc
 SHA512:
-  metadata.gz: 3b86f71da951714df9558c4918e6bcf36b29815d97aeb907e71212762566f089baaf6e6fcf514df6e3569c8bdec17b11e50690c0df3d803eb4079805132c45d1
-  data.tar.gz: 3c0d6cd34faa4fc5b2853775c890053e27761f5d7e1f9eed510323b487e33206c82b983ce93b07d53e7ab8a1b5c4bf1b12565ea0f45adb29f51b151e72d8076d
+  metadata.gz: c5bd9cf986be28aa07d504e771a2be2fef1260c7e0a76f9e6bf91b405c81331b51f0d5c7ca551b9080b4383d22de9391c7551e8343b03c8392a75130e7bb204b
+  data.tar.gz: b85287965ddcf34ee7f64e1dc857bc2106257fac8b927ea2bd1862249a5f935473a4bfe7a690aa013ab9d435b4272e4177e4da626e187790b61423b5403ef9a3

data/db/migrate/20140221062741_expand_log.rb

@@ -0,0 +1,11 @@
+class ExpandLog < ActiveRecord::Migration
+  def change
+    change_column(:tasksets, :setup_command, :text, limit: 4294967295)
+    change_column(:tasksets, :slave_command, :text, limit: 4294967295)
+    change_column(:tasksets, :log, :text, limit: 4294967295)
+    change_column(:trials, :stdout, :text, limit: 4294967295)
+    change_column(:trials, :stderr, :text, limit: 4294967295)
+    change_column(:worker_logs, :log, :text, limit: 4294967295)
+    change_column(:slaves, :log, :text, limit: 4294967295)
+  end
+end

data/db/migrate/20140225062300_delete_logs.rb

@@ -0,0 +1,19 @@
+class DeleteLogs < ActiveRecord::Migration
+  def change
+    change_table :slaves do |t|
+      t.remove :log
+    end
+
+    change_table :tasksets do |t|
+      t.remove :log
+    end
+
+    change_table :trials do |t|
+      t.remove :stdout, :stderr
+    end
+
+    change_table :worker_logs do |t|
+      t.remove :log
+    end
+  end
+end

data/db/schema.rb

@@ -9,21 +9,20 @@
 # from scratch. The latter is a flawed and unsustainable approach (the more migrations
 # you'll amass, the slower it'll run and the greater likelihood for issues).
 #
-# It's strongly recommended to check this file into your version control system.
+# It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20131105050718) do
+ActiveRecord::Schema.define(version: 20140225062300) do
 
-  create_table "slaves", :force => true do |t|
+  create_table "slaves", force: true do |t|
     t.string  "key"
     t.integer "taskset_id"
     t.string  "status"
-    t.text    "log"
   end
 
-  add_index "slaves", ["key"], :name => "index_slaves_on_key"
-  add_index "slaves", ["taskset_id"], :name => "index_slaves_on_taskset_id"
+  add_index "slaves", ["key"], name: "index_slaves_on_key"
+  add_index "slaves", ["taskset_id"], name: "index_slaves_on_taskset_id"
 
-  create_table "tasks", :force => true do |t|
+  create_table "tasks", force: true do |t|
     t.string  "key"
     t.integer "taskset_id"
     t.string  "status"
@@ -31,15 +30,15 @@ ActiveRecord::Schema.define(:version => 20131105050718) do
     t.string  "spec_file"
   end
 
-  add_index "tasks", ["key"], :name => "index_tasks_on_key"
-  add_index "tasks", ["status"], :name => "index_tasks_on_status"
-  add_index "tasks", ["taskset_id"], :name => "index_tasks_on_taskset_id"
+  add_index "tasks", ["key"], name: "index_tasks_on_key"
+  add_index "tasks", ["status"], name: "index_tasks_on_status"
+  add_index "tasks", ["taskset_id"], name: "index_tasks_on_taskset_id"
 
-  create_table "tasksets", :force => true do |t|
+  create_table "tasksets", force: true do |t|
     t.string   "key"
     t.string   "rsync_name"
-    t.text     "setup_command"
-    t.text     "slave_command"
+    t.text     "setup_command",            limit: 4294967295
+    t.text     "slave_command",            limit: 4294967295
     t.string   "worker_type"
     t.integer  "max_workers"
     t.integer  "max_trials"
@@ -49,34 +48,31 @@ ActiveRecord::Schema.define(:version => 20131105050718) do
     t.datetime "created_at"
     t.string   "status"
     t.datetime "finished_at"
-    t.text     "log"
   end
 
-  add_index "tasksets", ["created_at"], :name => "index_tasksets_on_created_at"
-  add_index "tasksets", ["key"], :name => "index_tasksets_on_key"
-  add_index "tasksets", ["rsync_name"], :name => "index_tasksets_on_rsync_name"
-  add_index "tasksets", ["status"], :name => "index_tasksets_on_status"
-  add_index "tasksets", ["taskset_class"], :name => "index_tasksets_on_taskset_class"
+  add_index "tasksets", ["created_at"], name: "index_tasksets_on_created_at"
+  add_index "tasksets", ["key"], name: "index_tasksets_on_key"
+  add_index "tasksets", ["rsync_name"], name: "index_tasksets_on_rsync_name"
+  add_index "tasksets", ["status"], name: "index_tasksets_on_status"
+  add_index "tasksets", ["taskset_class"], name: "index_tasksets_on_taskset_class"
 
-  create_table "trials", :force => true do |t|
+  create_table "trials", force: true do |t|
     t.string   "key"
     t.integer  "task_id"
     t.integer  "slave_id"
     t.datetime "started_at"
     t.datetime "finished_at"
     t.string   "status"
-    t.text     "stdout"
-    t.text     "stderr"
     t.integer  "passed"
     t.integer  "pending"
     t.integer  "failed"
   end
 
-  add_index "trials", ["key"], :name => "index_trials_on_key"
-  add_index "trials", ["slave_id"], :name => "index_trials_on_slave_id"
-  add_index "trials", ["task_id"], :name => "index_trials_on_task_id"
+  add_index "trials", ["key"], name: "index_trials_on_key"
+  add_index "trials", ["slave_id"], name: "index_trials_on_slave_id"
+  add_index "trials", ["task_id"], name: "index_trials_on_task_id"
 
-  create_table "worker_logs", :force => true do |t|
+  create_table "worker_logs", force: true do |t|
     t.string   "key"
     t.string   "worker_key"
     t.integer  "taskset_id"
@@ -84,11 +80,10 @@ ActiveRecord::Schema.define(:version => 20131105050718) do
     t.datetime "rsync_finished_at"
     t.datetime "setup_finished_at"
     t.datetime "finished_at"
-    t.text     "log"
   end
 
-  add_index "worker_logs", ["key"], :name => "index_worker_logs_on_key"
-  add_index "worker_logs", ["taskset_id"], :name => "index_worker_logs_on_taskset_id"
-  add_index "worker_logs", ["worker_key"], :name => "index_worker_logs_on_worker_key"
+  add_index "worker_logs", ["key"], name: "index_worker_logs_on_key"
+  add_index "worker_logs", ["taskset_id"], name: "index_worker_logs_on_taskset_id"
+  add_index "worker_logs", ["worker_key"], name: "index_worker_logs_on_worker_key"
 
 end

data/lib/rrrspec/server.rb

@@ -1,9 +1,13 @@
+require 'fileutils'
 require 'active_record'
 
 require 'rrrspec'
 require 'rrrspec/server/arbiter'
 require 'rrrspec/server/configuration'
+require 'rrrspec/server/daemonizer'
 require 'rrrspec/server/dispatcher'
+require 'rrrspec/server/json_constructor'
+require 'rrrspec/server/log_file_persister'
 require 'rrrspec/server/persistent_models'
 require 'rrrspec/server/persister'
 require 'rrrspec/server/worker_runner'

data/lib/rrrspec/server/cli.rb

@@ -12,73 +12,21 @@ module RRRSpec
         def setup(conf)
           RRRSpec.setup(conf, options[:config].split(':'))
         end
-
-        def log_exception
-          yield
-        rescue
-          RRRSpec.logger.error($!)
-          raise
-        end
-
-        def auto_rebirth
-          Signal.trap('TERM') do
-            Signal.trap('TERM', 'DEFAULT')
-            Process.kill('TERM', 0)
-          end
-          loop do
-            pid = Process.fork do
-              log_exception { yield }
-            end
-            Process.waitpid(pid)
-          end
-        end
-
-        def daemonize
-          return unless options[:daemonize]
-          pidfile = options[:pidfile]
-
-          if pidfile
-            pidfile = File.absolute_path(pidfile)
-            if File.exists?(pidfile)
-              pid = open(pidfile, 'r') { |f| f.read.strip }
-              if pid
-                begin
-                  if Process.kill(0, pid.to_i) == 1
-                    $stderr.puts "Daemon process already exists: #{pid}"
-                    exit 1
-                  end
-                rescue Errno::ESRCH
-                end
-              end
-            end
-          end
-
-          Process.daemon
-
-          if pidfile
-            open(pidfile, 'w') { |f| f.write(Process.pid.to_s) }
-
-            parent_pid = Process.pid
-            at_exit do
-              if Process.pid == parent_pid
-                File.delete(pidfile)
-              end
-            end
-          end
-        end
       end
 
       method_option :daemonize, type: :boolean
       method_option :pidfile, type: :string
+      method_option :user, type: :string
       desc 'server', 'Run RRRSpec as a server'
       def server
-        $0 = 'rrrspec server'
         setup(ServerConfiguration.new)
-        daemonize
-        auto_rebirth do
+        RRRSpec.configuration.daemonize = options[:daemonize] unless options[:daemonize] == nil
+        RRRSpec.configuration.pidfile = options[:pidfile] unless options[:pidfile] == nil
+        RRRSpec.configuration.user = options[:user] unless options[:user] == nil
+
+        RRRSpec::Server.daemonizer('rrrspec-server server') do
           ActiveRecord::Base.establish_connection(**RRRSpec.configuration.persistence_db)
           Thread.abort_on_exception = true
-          Thread.fork { RRRSpec.pacemaker(RSyncInfo, 60, 5) }
           Thread.fork { Dispatcher.work_loop }
           Thread.fork { Arbiter.work_loop }
           Thread.fork { Persister.work_loop }
@@ -86,15 +34,19 @@ module RRRSpec
         end
       end
 
+      method_option :'worker-type', type: :string
       method_option :daemonize, type: :boolean
       method_option :pidfile, type: :string
+      method_option :user, type: :string
       desc 'worker', 'Run RRRSpec as a worker'
       def worker
-        $0 = 'rrrspec worker'
         setup(WorkerConfiguration.new)
-        daemonize
-        auto_rebirth do
-          worker = Worker.create(RRRSpec.configuration.worker_type)
+        RRRSpec.configuration.daemonize = options[:daemonize] unless options[:daemonize] == nil
+        RRRSpec.configuration.pidfile = options[:pidfile] unless options[:pidfile] == nil
+        RRRSpec.configuration.user = options[:user] unless options[:user] == nil
+
+        RRRSpec::Server.daemonizer('rrrspec-server worker') do
+          worker = Worker.create(options[:'worker-type'] || RRRSpec.configuration.worker_type)
           worker_runner = WorkerRunner.new(worker)
           Thread.abort_on_exception = true
           Thread.fork { RRRSpec.pacemaker(worker, 60, 5) }

data/lib/rrrspec/server/configuration.rb

@@ -3,9 +3,10 @@ require 'facter'
 module RRRSpec
   module Server
     class ServerConfiguration < Configuration
-      attr_accessor :rsync_server, :rsync_dir, :rsync_options
       attr_accessor :persistence_db
+      attr_accessor :execute_log_text_path
       attr_accessor :json_cache_path
+      attr_accessor :daemonize, :pidfile, :user
 
       def initialize
         super()
@@ -15,8 +16,8 @@ module RRRSpec
       def check_validity
         validity = super
 
-        unless rsync_server and rsync_options and rsync_dir
-          $stderr.puts('The rsync options are not set')
+        unless execute_log_text_path
+          $stderr.puts('The path to save the log text should be set')
           validity = false
         end
 
@@ -30,11 +31,13 @@ module RRRSpec
     end
 
     class WorkerConfiguration < Configuration
+      attr_accessor :rsync_remote_path, :rsync_options
       attr_accessor :working_dir, :worker_type, :slave_processes
+      attr_accessor :daemonize, :pidfile, :user
 
       def initialize
         super()
-        @slave_processes = Facter.processorcount.to_i
+        @slave_processes = Facter.value(:processorcount).to_i
         @worker_type = 'default'
         @type = :worker
       end
@@ -42,6 +45,11 @@ module RRRSpec
       def check_validity
         validity = super
 
+        unless rsync_remote_path and rsync_options
+          $stderr.puts('The rsync options are not set')
+          validity = false
+        end
+
         unless working_dir and worker_type
           $stderr.puts('The worker options are not set')
           validity = false

data/lib/rrrspec/server/daemonizer.rb

@@ -0,0 +1,87 @@
+module RRRSpec
+  module Server
+    RESPAWN_INTERVAL_LIMIT_SEC = 30
+
+    def self.daemonizer(process_name, &block)
+      $0 = process_name
+      return block.call unless should_daemonize?
+
+      pidfile = File.absolute_path(RRRSpec.configuration.pidfile || File.join("/var/run", "#{process_name}.pid"))
+      check_pidfile(pidfile)
+
+      Process.daemon
+      File.write(pidfile, Process.pid.to_s)
+      File.umask(0)
+      setup_signal_handlers
+      setup_atexit_handlers(pidfile)
+      monitor_fork do
+        Process::Sys.setuid(RRRSpec.configuration.user) if RRRSpec.configuration.user
+        block.call
+      end
+    end
+
+    def self.should_daemonize?
+      RRRSpec.configuration.daemonize == nil || RRRSpec.configuration.daemonize
+    end
+
+    def self.should_monitor?
+      RRRSpec.configuration.monitor == nil || RRRSpec.configuration.monitor
+    end
+
+    def self.check_pidfile(pidfile)
+      if File.exist?(pidfile)
+        if File.readable?(pidfile)
+          pid = File.read(pidfile).to_i
+          begin
+            Process.kill(0, pid)
+            raise "Pid(#{pid}) is running"
+          rescue Errno::EPERM
+            raise "Pid(#{pid}) is running"
+          rescue Errno::ESRCH
+          end
+        else
+          raise "Cannot access #{pidfile}"
+        end
+
+        unless File.writable?(pidfile)
+          raise "Cannot access #{pidfile}"
+        end
+      else
+        unless File.writable?(File.dirname(pidfile))
+          raise "Cannot access #{pidfile}"
+        end
+      end
+    end
+
+    def self.setup_signal_handlers
+      # Propagate the TERM signal to the children
+      Signal.trap('TERM') do
+        Signal.trap('TERM', 'DEFAULT')
+        Process.kill('TERM', 0)
+      end
+    end
+
+    def self.setup_atexit_handlers(pidfile)
+      # Delete pid at exit
+      current_pid = Process.pid
+      at_exit do
+        # Since at_exit handlers are inherited by child processes, it is the
+        # case that the handlers are invoked in the child processes. This guard
+        # is needed to avoid this.
+        File.delete(pidfile) if Process.pid == current_pid
+      end
+    end
+
+    def self.monitor_fork(&block)
+      loop do
+        started_at = Time.now
+        pid = Process.fork(&block)
+        Process.waitpid(pid)
+        break unless should_monitor?
+        if Time.now - started_at < RESPAWN_INTERVAL_LIMIT_SEC
+          sleep RESPAWN_INTERVAL_LIMIT_SEC
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/json_constructor.rb

@@ -0,0 +1,187 @@
+module RRRSpec
+  module Server
+    module JSONConstructor
+      module TasksetJSONConstructor
+        extend ActiveSupport::Concern
+        include ActiveModel::Serializers::JSON
+
+        def as_nodetail_json
+          as_json(except: :id, methods: :log)
+        end
+
+        def as_short_json
+          h = as_json(except: :id, methods: :log)
+          h['slaves'] = slaves.map { |slave| slave.as_json(only: :key) }
+          h['tasks'] = tasks.map { |task| task.as_json(only: :key) }
+          h['worker_logs'] = worker_logs.map { |worker_log| worker_log.as_json(only: :key) }
+          h
+        end
+
+        def as_full_json
+          h = as_json(except: :id, methods: :log)
+          h['slaves'] = slaves.map(&:as_full_json)
+          h['tasks'] = tasks.map(&:as_full_json)
+          h['worker_logs'] = worker_logs.map(&:as_full_json)
+          h
+        end
+
+        def as_json_for_index
+          {
+            'id' => id,
+            'key' => key,
+            'rsync_name' => rsync_name,
+            'setup_command' => setup_command,
+            'slave_command' => slave_command,
+            'worker_type' => worker_type,
+            'max_workers' => max_workers,
+            'max_trials' => max_trials,
+            'taskset_class' => taskset_class,
+            'created_at' => created_at,
+            'status' => status,
+            'finished_at' => finished_at,
+          }
+        end
+
+        def as_json_for_result_page
+          as_json_for_index.merge(
+            'tasks' => tasks.map(&:as_json_for_result_page),
+          )
+        end
+
+        def as_summary_json
+          statuses = tasks.pluck(:status)
+          groups = statuses.group_by { |status| status }
+          groups.default = []
+
+          {
+            'id' => id,
+            'key' => key,
+            'status' => status,
+            'created_at' => created_at,
+            'finished_at' => finished_at,
+            'task_count' => statuses.count,
+            'passed_count' => groups['passed'].size,
+            'pending_count' => groups['pending'].size,
+            'failed_count' => groups['failed'].size,
+          }
+        end
+      end
+
+      module TaskJSONConstructor
+        extend ActiveSupport::Concern
+        include ActiveModel::Serializers::JSON
+
+        def as_short_json
+          h = as_json(except: [:id, :taskset_id, :trials],
+                      include: { 'taskset' => { only: :key } })
+          h['trials'] = trials.map { |trial| trial.as_json(only: :key) }
+          h
+        end
+
+        def as_full_json
+          h = as_json(except: [:id, :taskset_id, :trials],
+                      include: { 'taskset' => { only: :key } })
+          h['trials'] = trials.map(&:as_full_json)
+          h
+        end
+
+        def as_json_for_result_page
+          {
+            'id' => id,
+            'key' => key,
+            'status' => status,
+            'spec_path' => spec_file,
+            'estimate_sec' => estimate_sec,
+            'trials' => trials.map(&:as_json_for_result_page),
+          }
+        end
+      end
+
+      module TrialJSONConstructor
+        extend ActiveSupport::Concern
+        include ActiveModel::Serializers::JSON
+
+        def as_short_json
+          as_full_json
+        end
+
+        def as_full_json
+          as_json(except: [:id, :task_id, :slave_id],
+                  include: { 'slave' => { only: :key }, 'task' => { only: :key } },
+                  methods: [:stdout, :stderr])
+        end
+
+        def as_json_for_result_page
+          {
+            'id' => id,
+            'key' => key,
+            'task_id' => task_id,
+            'slave_id' => slave_id,
+            'started_at' => started_at,
+            'finished_at' => finished_at,
+            'status' => status,
+            'passed' => passed,
+            'pending' => pending,
+            'failed' => failed,
+          }
+        end
+      end
+
+      module WorkerLogJSONConstructor
+        extend ActiveSupport::Concern
+        include ActiveModel::Serializers::JSON
+
+        def as_short_json
+          as_full_json
+        end
+
+        def as_full_json
+          as_json(except: [:id, :taskset_id, :worker_key],
+                  include: { 'taskset' => { only: :key } },
+                  methods: [:worker, :log])
+        end
+
+        def worker
+          { 'key' => worker_key }
+        end
+
+        def as_json_for_result_page
+          {
+            'id' => id,
+            'worker_name' => worker_key,
+            'started_at' => started_at,
+            'rsync_finished_at' => rsync_finished_at,
+            'setup_finished_at' => setup_finished_at,
+            'rspec_finished_at' => finished_at,
+            'log' => log.to_s,
+          }
+        end
+      end
+
+      module SlaveJSONConstructor
+        extend ActiveSupport::Concern
+        include ActiveModel::Serializers::JSON
+
+        def as_short_json
+          as_full_json
+        end
+
+        def as_full_json
+          as_json(except: [:id, :taskset_id],
+                  include: { 'trials' => { only: :key } },
+                  methods: [:log])
+        end
+
+        def as_json_for_result_page
+          {
+            'id' => id,
+            'name' => key,
+            'status' => status,
+            'trials' => trials.map { |trial| { id: trial.id, key: trial.key } },
+            'log' => log.to_s,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/log_file_persister.rb

@@ -0,0 +1,66 @@
+module RRRSpec
+  module Server
+    # LogFilePersister
+    #
+    #     class Taskset < ActiveRecord::Base
+    #       include LogFilePersister
+    #       save_as_file :log, suffix: 'log'
+    #     end
+    #
+    # Requirements:
+    #
+    # * Have a property named 'key'
+    # * Set RRRSpec.configuration.execute_log_text_path
+    # * Have a singleton method :after_save
+    #
+    # Provides:
+    #
+    # * Methods named 'log', 'log=', 'log_log_path'.
+    # * 'after_save' hook that persists a log to the file pointed by log_log_path.
+    # * Persist the value of the 'log' to File.join(execute_log_text_path, "#{key.gsub(/\//, '_').gsub(/:/, '/')}_log.log") if it is dirty.
+    module LogFilePersister
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def save_as_file(name, suffix: nil)
+          suffix = "_#{suffix}" if suffix
+          instance_variable_set("@#{name}_dirty", false)
+          define_method("#{name}_log_path") do
+            return nil unless send(:key)
+            File.join(
+              RRRSpec.configuration.execute_log_text_path,
+              "#{send(:key).gsub(/\//, '_').gsub(/:/, '/')}#{suffix}.log"
+            )
+          end
+
+          define_method(name) do
+            contents = instance_variable_get("@#{name}")
+            if !contents
+              log_path = send("#{name}_log_path")
+              if log_path
+                contents = File.exist?(log_path) ?  File.read(log_path) : nil
+                instance_variable_set("@#{name}", contents) if contents
+              end
+            end
+            contents
+          end
+
+          define_method("#{name}=") do |val|
+            if send(name) != val
+              instance_variable_set("@#{name}_dirty", true)
+              instance_variable_set("@#{name}", val)
+            end
+          end
+
+          after_save do
+            if instance_variable_get("@#{name}")
+              log_path = send("#{name}_log_path")
+              FileUtils.mkdir_p(File.dirname(log_path))
+              File.open(log_path, 'w') { |fp| fp.write(send(name)) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rrrspec/server/persistent_models.rb

@@ -2,109 +2,51 @@ module RRRSpec
   module Server
     module Persistence
       class Taskset < ActiveRecord::Base
-        include ActiveModel::Serializers::JSON
-
+        include JSONConstructor::TasksetJSONConstructor
+        include LogFilePersister
         has_many :worker_logs
         has_many :slaves
         has_many :tasks
+        save_as_file :log, suffix: 'log'
 
-        scope :full, includes(
-          :tasks => [{:trials => [:task, :slave]}, :taskset],
-          :slaves => [:trials],
-          :worker_logs => [:taskset]
-        )
-
-        def as_nodetail_json
-          as_json(except: :id)
-        end
-
-        def as_short_json
-          h = as_json(except: :id)
-          h['slaves'] = slaves.map { |slave| slave.as_json(only: :key) }
-          h['tasks'] = tasks.map { |task| task.as_json(only: :key) }
-          h['worker_logs'] = worker_logs.map { |worker_log| worker_log.as_json(only: :key) }
-          h
+        def self.full
+          includes(
+            :tasks => [{:trials => [:task, :slave]}, :taskset],
+            :slaves => [:trials],
+            :worker_logs => [:taskset]
+          )
         end
 
-        def as_full_json
-          h = as_json(except: :id)
-          h['slaves'] = slaves.map(&:as_full_json)
-          h['tasks'] = tasks.map(&:as_full_json)
-          h['worker_logs'] = worker_logs.map(&:as_full_json)
-          h
-        end
+        scope :by_redis_model, -> (_) { where(key: _.key) }
       end
 
       class Task < ActiveRecord::Base
-        include ActiveModel::Serializers::JSON
-
+        include JSONConstructor::TaskJSONConstructor
         belongs_to :taskset
         has_many :trials
-
-        def as_short_json
-          h = as_json(except: [:id, :taskset_id, :trials],
-                      include: { 'taskset' => { only: :key } })
-          h['trials'] = trials.map { |trial| trial.as_json(only: :key) }
-          h
-        end
-
-        def as_full_json
-          h = as_json(except: [:id, :taskset_id, :trials],
-                      include: { 'taskset' => { only: :key } })
-          h['trials'] = trials.map(&:as_full_json)
-          h
-        end
       end
 
       class Trial < ActiveRecord::Base
-        include ActiveModel::Serializers::JSON
-
+        include JSONConstructor::TrialJSONConstructor
+        include LogFilePersister
         belongs_to :task
         belongs_to :slave
-
-        def as_full_json
-          as_json(except: [:id, :task_id, :slave_id],
-                  include: { 'slave' => { only: :key }, 'task' => { only: :key } })
-        end
-
-        def as_short_json
-          as_full_json
-        end
+        save_as_file :stdout, suffix: 'stdout'
+        save_as_file :stderr, suffix: 'stderr'
       end
 
       class WorkerLog < ActiveRecord::Base
-        include ActiveModel::Serializers::JSON
-
+        include JSONConstructor::WorkerLogJSONConstructor
+        include LogFilePersister
         belongs_to :taskset
-
-        def as_full_json
-          as_json(except: [:id, :taskset_id, :worker_key],
-                  include: { 'taskset' => { only: :key } },
-                  methods: :worker)
-        end
-
-        def as_short_json
-          as_full_json
-        end
-
-        def worker
-          { 'key' => worker_key }
-        end
+        save_as_file :log, suffix: 'worker_log'
       end
 
       class Slave < ActiveRecord::Base
-        include ActiveModel::Serializers::JSON
-
+        include JSONConstructor::SlaveJSONConstructor
+        include LogFilePersister
         has_many :trials
-
-        def as_full_json
-          as_json(except: [:id, :taskset_id],
-                  include: { 'trials' => { only: :key } })
-        end
-
-        def as_short_json
-          as_full_json
-        end
+        save_as_file :log, suffix: 'slave_log'
       end
     end
   end

data/lib/rrrspec/server/persister.rb

@@ -62,17 +62,20 @@ module RRRSpec
             p_slave
           end
           Persistence::Slave.import(p_slaves)
+          p_slaves.each { |p_slave| p_slave.run_callbacks(:save) {} }
         end
 
         ActiveRecord::Base.transaction do
-          Persistence::Task.import(taskset.tasks.map do |task|
+          p_tasks = taskset.tasks.map do |task|
             h = task.to_h
             h.delete('taskset')
             h.delete('trials')
             p_task = Persistence::Task.new(h)
             p_task.taskset_id = p_taskset
             p_task
-          end)
+          end
+          Persistence::Task.import(p_tasks)
+          p_tasks.each { |p_task| p_task.run_callbacks(:save) {} }
         end
 
         p_slaves = {}
@@ -91,14 +94,16 @@ module RRRSpec
               p_trial = Persistence::Trial.new(h)
               p_trial.task_id = p_task
               p_trial.slave_id = p_slaves[slave_key]
+
               p_trials << p_trial
             end
           end
           Persistence::Trial.import(p_trials)
+          p_trials.each { |p_trial| p_trial.run_callbacks(:save) {} }
         end
 
         ActiveRecord::Base.transaction do
-          Persistence::WorkerLog.import(taskset.worker_logs.map do |worker_log|
+          p_worker_logs = taskset.worker_logs.map do |worker_log|
             h = worker_log.to_h
             h['worker_key'] = h['worker']['key']
             h.delete('worker')
@@ -106,7 +111,9 @@ module RRRSpec
             p_worker_log = Persistence::WorkerLog.new(h)
             p_worker_log.taskset_id = p_taskset
             p_worker_log
-          end)
+          end
+          Persistence::WorkerLog.import(p_worker_logs)
+          p_worker_logs.each { |p_worker_log| p_worker_log.run_callbacks(:save) {} }
         end
       end
 

data/lib/rrrspec/server/version.rb

@@ -1,5 +1,5 @@
 module RRRSpec
   module Server
-    VERSION = "0.2.0"
+    VERSION = "0.2.1"
   end
 end

data/lib/rrrspec/server/worker_runner.rb

@@ -24,8 +24,8 @@ module RRRSpec
 
         working_path = File.join(RRRSpec.configuration.working_dir, taskset.rsync_name)
         FileUtils.mkdir_p(working_path) unless Dir.exists?(working_path)
-        remote_path = "#{RSyncInfo.rsync_server}:#{File.join(RSyncInfo.rsync_dir, taskset.rsync_name)}"
-        command = "rsync #{RSyncInfo.rsync_options} #{remote_path}/ #{working_path}"
+        remote_path = File.join(RRRSpec.configuration.rsync_remote_path, taskset.rsync_name)
+        command = "rsync #{RRRSpec.configuration.rsync_options} #{remote_path}/ #{working_path}"
 
         pid, out_rd, err_rd = execute_with_logs(working_path, command, {})
         log_to_logger(logger, out_rd, err_rd)
@@ -150,7 +150,7 @@ module RRRSpec
         (pid_to_slave_number.keys + [cancel_watcher_pid]).each do |pid|
           begin
             Process.kill("-TERM", pid)
-          rescue Errno::ESRCH
+          rescue Errno::ESRCH, Errno::EPERM
           end
         end
 
@@ -171,7 +171,7 @@ module RRRSpec
         `ps aux | grep "rrrspec slave" | grep -v grep | awk '{print $2}'`.split("\n").map(&:to_i).each do |pid|
           begin
             Process.kill("KILL", pid)
-          rescue Errno::ESRCH
+          rescue Errno::ESRCH, Errno::EPERM
           end
         end
       end

data/rrrspec-server.gemspec

@@ -29,10 +29,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "sqlite3"
   spec.add_development_dependency "timecop"
-  spec.add_dependency "activerecord", "~> 3.0"
-  spec.add_dependency "activerecord-import", "~> 0.3.1"
+  spec.add_dependency "activerecord", "~> 4.0.2"
+  spec.add_dependency "activerecord-import", "~> 0.4.1"
   spec.add_dependency "activesupport"
   spec.add_dependency "bundler"
+  spec.add_dependency "facter"
   spec.add_dependency "redis"
   spec.add_dependency "rrrspec-client"
   spec.add_dependency "thor"

data/spec/rrrspec/server/log_file_persister_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+module RRRSpec
+  module Server
+    class TestClass
+      include LogFilePersister
+      def self.after_save(&block); end
+      save_as_file :log, suffix: 'log'
+
+      attr_accessor :key
+    end
+
+    describe LogFilePersister do
+      before do
+        RRRSpec.configuration = ServerConfiguration.new
+        RRRSpec.configuration.execute_log_text_path = '/tmp/log_path'
+      end
+
+      describe '#log_log_path' do
+        subject { TestClass.new }
+
+        it 'subsitutes colons in the key' do
+          subject.key = 'rrrspec:test'
+          expect(subject.log_log_path).to eq('/tmp/log_path/rrrspec/test_log.log')
+        end
+
+        it 'subsitutes slashes in the key' do
+          subject.key = 'rrrspec:test/path'
+          expect(subject.log_log_path).to eq('/tmp/log_path/rrrspec/test_path_log.log')
+        end
+      end
+    end
+  end
+end

data/spec/rrrspec/server/persistent_models_spec.rb

@@ -5,8 +5,9 @@ module RRRSpec
     module Persistence
       describe Taskset do
         before do
-          RRRSpec.configuration = RRRSpec::Configuration.new
+          RRRSpec.configuration = ServerConfiguration.new
           RRRSpec.configuration.redis = @redis
+          RRRSpec.configuration.execute_log_text_path = Dir.mktmpdir
         end
 
         before do

data/spec/rrrspec/server/persister_spec.rb

@@ -4,8 +4,9 @@ module RRRSpec
   module Server
     describe Persister do
       before do
-        RRRSpec.configuration = Configuration.new
+        RRRSpec.configuration = ServerConfiguration.new
         RRRSpec.configuration.redis = @redis
+        RRRSpec.configuration.execute_log_text_path = Dir.mktmpdir
       end
 
       before do
@@ -13,6 +14,10 @@ module RRRSpec
           RRRSpec.finished_fullset
       end
 
+      after do
+        FileUtils.remove_entry_secure(RRRSpec.configuration.execute_log_text_path)
+      end
+
       describe '.persist' do
         let(:task_json) do
           {
@@ -34,6 +39,16 @@ module RRRSpec
           }
         end
 
+        def log_path_of(filename)
+          File.join(RRRSpec.configuration.execute_log_text_path, filename)
+        end
+
+        let(:taskset_log_name)  { @taskset.key.gsub(/\//, '_').gsub(/:/, '/') + "_log.log" }
+        let(:worker_log_name)   { @worker_log.key.gsub(/\//, '_').gsub(/:/, '/') + "_worker_log.log" }
+        let(:slave_log_name)    { @slave.key.gsub(/\//, '_').gsub(/:/, '/') + "_slave_log.log" }
+        let(:trial_stdout_name) { @trial.key.gsub(/\//, '_').gsub(/:/, '/') + "_stdout.log" }
+        let(:trial_stderr_name) { @trial.key.gsub(/\//, '_').gsub(/:/, '/') + "_stderr.log" }
+
         def eq_json(actual, expected)
           expect(
             JSON.parse(JSON.generate(actual))
@@ -69,6 +84,28 @@ module RRRSpec
           check_persistence
         end
 
+        context 'when logs are too long' do
+          before do
+            RRRSpec.redis.hset(@trial.key, 'stdout', 'out' * 30000)
+          end
+
+          it 'saves full logs to the files' do
+            Persister.persist(@taskset)
+            expect(File.read(log_path_of(trial_stdout_name))).to eq(@trial.stdout)
+          end
+        end
+
+
+        it 'creates log text files' do
+          Persister.persist(@taskset)
+
+          expect(File.read(log_path_of(taskset_log_name))).to eq(@taskset.log)
+          expect(File.read(log_path_of(worker_log_name))).to eq(@worker_log.log)
+          expect(File.read(log_path_of(slave_log_name))).to eq(@slave.log)
+          expect(File.read(log_path_of(trial_stdout_name))).to eq(@trial.stdout)
+          expect(File.read(log_path_of(trial_stderr_name))).to eq(@trial.stderr)
+        end
+
         context "trial is finished after the taskset's finish" do
           before do
             @late_trial = Trial.create(@task, @slave)

metadata

@@ -1,195 +1,209 @@
 --- !ruby/object:Gem::Specification
 name: rrrspec-server
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Masaya Suzuki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 4.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 4.0.2
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rrrspec-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Execute RSpec in a distributed manner
@@ -200,17 +214,22 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .rspec
+- ".rspec"
 - Gemfile
 - Rakefile
 - bin/rrrspec-server
 - db/migrate/20131105050718_create_tables.rb
+- db/migrate/20140221062741_expand_log.rb
+- db/migrate/20140225062300_delete_logs.rb
 - db/schema.rb
 - lib/rrrspec/server.rb
 - lib/rrrspec/server/arbiter.rb
 - lib/rrrspec/server/cli.rb
 - lib/rrrspec/server/configuration.rb
+- lib/rrrspec/server/daemonizer.rb
 - lib/rrrspec/server/dispatcher.rb
+- lib/rrrspec/server/json_constructor.rb
+- lib/rrrspec/server/log_file_persister.rb
 - lib/rrrspec/server/persistent_models.rb
 - lib/rrrspec/server/persister.rb
 - lib/rrrspec/server/version.rb
@@ -219,6 +238,7 @@ files:
 - spec/fixture.rb
 - spec/rrrspec/server/arbiter_spec.rb
 - spec/rrrspec/server/dispatcher_spec.rb
+- spec/rrrspec/server/log_file_persister_spec.rb
 - spec/rrrspec/server/persistent_models_spec.rb
 - spec/rrrspec/server/persister_spec.rb
 - spec/spec_helper.rb
@@ -233,17 +253,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Execute RSpec in a distributed manner
@@ -251,6 +271,7 @@ test_files:
 - spec/fixture.rb
 - spec/rrrspec/server/arbiter_spec.rb
 - spec/rrrspec/server/dispatcher_spec.rb
+- spec/rrrspec/server/log_file_persister_spec.rb
 - spec/rrrspec/server/persistent_models_spec.rb
 - spec/rrrspec/server/persister_spec.rb
 - spec/spec_helper.rb