dtr 1.0.0 → 1.1.0

data/lib/dtr/agent/process_root.rb

@@ -0,0 +1,23 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    class ProcessRoot
+      def initialize(*args)
+        DTR.root = Dir.pwd
+      end
+    end
+  end
+end

data/lib/dtr/agent/rails_ext.rb

@@ -0,0 +1,57 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    module RailsExt
+      module DatabaseInitializer
+        def preparing_database_command
+          dtr_database_config_exists = File.exist?('config/database.yml.dtr')
+          default_database_config_exists = File.exist?('config/database.yml')
+
+          if !dtr_database_config_exists && !default_database_config_exists
+            DTR.info("No config/database.yml.dtr and config/database.yml exists, bypass database initialization.")
+            return
+          end
+
+          if dtr_database_config_exists
+            DTR.info("Found config/database.yml.dtr, use it as database configuration")
+            FileUtils.cp('config/database.yml.dtr', 'config/database.yml')
+          end
+          DTR.info("Clean databases")
+          Cmd.execute("rake --trace db:drop DTR_RUNNER_NAME=#{ENV['DTR_RUNNER_NAME']}", :error_output_log_level => :debug)
+
+          # Counldn't add --trace here, for Test::Unit detected --trace as a invalid option, don't know why
+          "rake db:create db:migrate db:test:prepare DTR_RUNNER_NAME=#{ENV['DTR_RUNNER_NAME']}"
+        end
+      end
+
+      module WorkingEnvExt
+        include DatabaseInitializer
+
+        def self.included(base)
+          base.alias_method_chain :setup_environment, :preparing_database
+        end
+
+        def setup_environment_with_preparing_database
+          if setup_environment_command.blank? && File.directory?('config')
+            DTR.debug("No setup environment command found but found 'config' directory, try default preparing database command")
+            self[:agent_env_setup_cmd] = preparing_database_command
+          end
+          setup_environment_without_preparing_database
+        end
+      end
+    end
+  end
+end

data/lib/dtr/agent/runner.rb

@@ -14,65 +14,45 @@
 
 module DTR
   module Agent
-    class Runner
+    class Runner < ProcessRoot
       include DRbUndumped
       include Service::Runner
 
-      def self.start(name, env)
-        self.new(name, env).start
-        DRb.thread.join if DRb.thread
-      end
-
-      attr_reader :name, :identifier
+      attr_reader :name
 
-      def initialize(name, env)
+      def initialize(name)
+        super
         @name = name
-        @identifier = env[:identifier]
-        @env = env
       end
       
       def start
         #start service first, so that all logs can be sync with master process
         start_service
-        DTR.info("=> Starting runner #{name} at #{Dir.pwd}, pid: #{Process.pid}")
-        init_environment
-        provide
-        DTR.info {"=> Runner #{name} provided"}
-      rescue Exception
-        DTR.error($!.message)
-        DTR.error($!.backtrace.join("\n"))
-      end
 
-      def init_environment
-        DTR.info {"#{name}: Initialize working environment..."}
         ENV['DTR_RUNNER_NAME'] = name
-
-        @env[:libs].select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
-          $LOAD_PATH << lib
-          DTR.debug {"#{name}: appended lib: #{lib}"}
-        end
-        DTR.info {"#{name}: libs loaded"}
-        DTR.debug {"#{name}: $LOAD_PATH: #{$LOAD_PATH.inspect}"}
-
-        @env[:files].each do |f|
-          begin
-            load f unless f =~ /^-/
-            DTR.debug {"#{name}: loaded #{f}"}
-          rescue LoadError => e
-            DTR.error {"#{name}: No such file to load -- #{f}"}
-            DTR.debug {"Environment: #{@env}"}
+        DTR.info "Start #{self} at #{DTR.root}, pid: #{Process.pid}"
+        DTR.configuration.working_env.load_environment do
+          provide
+          DTR.info {"=> Runner provided"}
+          while DTR.configuration.runners_should_be_working?
+            sleep(1)
           end
         end
-        DTR.info {"#{name}: test files loaded"}
+      rescue
+        DTR.error($!.message)
+        DTR.error($!.backtrace.join("\n"))
+      ensure
+        #make sure exit process for drb may cause this process hang on
+        exit!(0)
       end
 
       def run(test, result, &progress_block)
-        DTR.debug {"#{name}: running #{test}..."}
+        DTR.debug {"running #{test}..."}
         Agent::TestCase.new(test, result, &progress_block).run
-        DTR.debug {"#{name}: done #{test}"}
+        DTR.debug {"done #{test}"}
       ensure
         provide
-        DTR.debug {"=> Runner #{name} provided"}
+        DTR.debug {"=> Runner provided"}
       end
 
       def provide

data/lib/dtr/agent/sync_codebase.rb

@@ -17,27 +17,18 @@ module DTR
     module WorkingEnvExt
       include SyncService
 
-      def self.included(base)
-        base.alias_method_chain :setup_env, :sync_codebase
-        base.alias_method_chain :working_dir, :sync_codebase
-      end
-
-      def setup_env_with_sync_codebase(setup_env_cmd)
-        unless same_working_dir_with_master_process?
-          Dir.chdir(working_dir_without_sync_codebase) do
-            sync_codebase
+      def synchronize_for(runners)
+        Dir.chdir(base_dir) do
+          sync_codebase do
+            FileUtils.rm_rf(package_name)
+            do_work(unpackage_cmd)
+            runners.each do |runner_name|
+              dir = File.expand_path escape_dir(runner_name)
+              FileUtils.rm_rf(dir)
+              FileUtils.cp_r(package_name, dir)
+            end
           end
         end
-        setup_env_without_sync_codebase(setup_env_cmd)
-      end
-
-      def working_dir_with_sync_codebase
-        same_working_dir_with_master_process? ? Dir.pwd : File.join(working_dir_without_sync_codebase, package_name)
-      end
-
-      private
-      def same_working_dir_with_master_process?
-        self[:host] == Socket.gethostname && self[:pwd] == Dir.pwd
       end
     end
   end

data/lib/dtr/agent/sync_logger.rb

@@ -61,6 +61,7 @@ module DTR
       private
       def with_decorating_message(level, msg, &block)
         raise 'Should not use block to send log remotely' if block_given?
+        msg = "#{ENV['DTR_RUNNER_NAME']}: #{msg}"if ENV['DTR_RUNNER_NAME']
         @logger.send(level, decorate_message(msg))
       end
     end

data/lib/dtr/agent/worker.rb

@@ -15,74 +15,26 @@
 module DTR
 
   module Agent
-    # Worker works during one dtr test task running.
-    # Worker manages Herald & Runner processes life cycle.
+    # Worker watchs runner processes.
     class Worker
-      def initialize(runner_names, agent_env_setup_cmd)
-        @runner_names = runner_names.is_a?(Array) ? runner_names : [runner_names.to_s]
-        @agent_env_setup_cmd = agent_env_setup_cmd
-        @runner_pids = []
-        @herald = nil
-        @working_env_key = :working_env
-        @env_store = EnvStore.new
+      def initialize
+        @runners_group = ThreadGroup.new
       end
 
-      def launch
-        DTR.info {"=> Agent worker started at: #{Dir.pwd}, pid: #{Process.pid}"}
-        setup
-        begin
-          run
-        ensure
-          teardown
-          DTR.info {"Agent worker is dieing"}
+      def watch_runners
+        DTR.configuration.agent_runners.each do |name|
+          runner_thread = Thread.start { DTR.run_script("DTR::Agent::Runner.new(#{name.inspect}).start") }
+          runner_thread[:runner_name] = name
+          @runners_group.add runner_thread
         end
-      end
-
-      private
-      def setup
-        @env_store[@working_env_key] = nil
-      end
-
-      def teardown
-        unless @runner_pids.blank?
-          @runner_pids.each{ |pid| DTR.kill_process pid }
-          DTR.info {"=> All runners(#{@runner_pids.join(", ")}) were killed." }
-          @runner_pids = []
-        end
-        if @herald
-          DTR.kill_process @herald
-          @herald = nil
-          DTR.info {"=> Herald is killed."}
-        end
-      end
-
-      def run
-        herald
-        runners
-        DTR.info {"=> All agent worker sub processes exited."}
-      end
-
-      def herald
-        @herald = DTR.fork_process { Herald.new @working_env_key, @agent_env_setup_cmd, @runner_names }
-        Process.waitpid @herald
-        exit(-1) unless $?.exitstatus == 0
-      end
 
-      def runners
-        working_env = @env_store[@working_env_key]
+        yield
 
-        @runner_names.each do |name|
-          @runner_pids << DTR.fork_process {
-            at_exit {
-              # exit anyway, for DRb may hang on the process to be a deadwalk
-              exit!
-            }
-            working_env.within do
-              Runner.start name, working_env
-            end
-          }
+        while @runners_group.list.length > 0
+          alive_runners = @runners_group.list.collect {|r| r[:runner_name]}
+          DTR.info { "Waiting for #{alive_runners.join(', ').downcase} shutdown" }
+          sleep 1
         end
-        Process.waitall
       end
     end
   end

data/lib/dtr/agent/working_env_ext.rb

@@ -14,33 +14,67 @@
 
 module DTR
   module Agent
-    # Provides working environment setup beheaviours for Herald & Runner process
     module WorkingEnvExt
-      def within
-        ENV['DTR_MASTER_ENV'] = self[:dtr_master_env]
+      def base_dir
+        return @base_dir if defined?(@base_dir)
+        project_specific_len = 20
+        project_name = self[:pwd].length > project_specific_len ? self[:pwd][-project_specific_len..-1] : self[:pwd]
+        @base_dir = File.expand_path FileUtils.mkdir_p(File.join(escape_dir(self[:host]), escape_dir(project_name)))
+      end
+
+      def load_environment(&block)
+        working_dir = FileUtils.mkdir_p(File.join(base_dir, escape_dir(ENV['DTR_RUNNER_NAME'])))
         Dir.chdir(working_dir) do
-          yield
+          DTR.info "Loading environment at #{Dir.pwd}"
+
+          ENV['DTR_MASTER_ENV'] = dtr_master_env
+
+          setup_environment
+
+          append_libs_to_load_path
+          require_files
+
+          block.call
         end
       end
 
-      def setup_env(setup_env_cmd)
-        within do
-          Cmd.execute(setup_env_cmd || self[:agent_env_setup_cmd])
+      private
+
+      def setup_environment_command
+        DTR.configuration.agent_env_setup_cmd || self.agent_env_setup_cmd
+      end
+
+      def setup_environment
+        unless Cmd.execute(setup_environment_command)
+          raise "Stopped for setup working environment failed."
         end
       end
 
-      private
-      def escape(str)
-        str.gsub(/[^a-zA-Z0-9]/, '_')
+      def escape_dir(str)
+        str.to_s.gsub(/[^a-zA-Z0-9]/, '_')
       end
 
-      def working_dir
-        return @working_dir if defined?(@working_dir)
-        project_specific_len = 20
-        project_name = self[:pwd].length > project_specific_len ? self[:pwd][-project_specific_len..-1] : self[:pwd]
-        @working_dir = File.join(escape(self[:host]), escape(project_name))
-        FileUtils.mkdir_p(@working_dir)
-        @working_dir
+      def append_libs_to_load_path
+        libs.select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
+          $LOAD_PATH << lib
+          DTR.debug {"appended lib: #{lib}"}
+        end
+        DTR.info {"libs appended"}
+        DTR.debug {"$LOAD_PATH: #{$LOAD_PATH.inspect}"}
+      end
+
+      def require_files
+        files.each do |f|
+          begin
+            #use require instead of load to avoid load file twice.
+            require f unless f =~ /^-/
+            DTR.debug {"required #{f}"}
+          rescue LoadError => e
+            DTR.error {"No such file to load -- #{f}"}
+            DTR.debug {"Environment: #{self}"}
+          end
+        end
+        DTR.info {"test files loaded"}
       end
     end
   end

data/lib/dtr/agent/working_status.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    module WorkingStatus
+      WORKING_STATUS_KEY = :runners_working_status
+      def runners_should_be_working
+        @store[WORKING_STATUS_KEY] = Time.now
+      end
+
+      def runners_should_be_working?
+        if time = @store[WORKING_STATUS_KEY]
+          (time - Time.now) <= follower_listen_heartbeat_timeout
+        end
+      end
+
+      def agent_is_going_to_sleep
+        @store[WORKING_STATUS_KEY] = nil
+      end
+    end
+  end
+end

data/lib/dtr/facade.rb

@@ -15,13 +15,8 @@
 module DTR
   module Facade
     def start_agent
-      launch_agent(DTR_AGENT_OPTIONS[:runners], DTR_AGENT_OPTIONS[:agent_env_setup_cmd])
-    end
-
-    def launch_agent(names, setup=nil)
       require 'dtr/agent'
-      names = names || "DTR(#{Time.now})"
-      DTR::Agent.start(names, setup)
+      DTR::Agent.start
     end
 
     def lib_path
@@ -29,37 +24,44 @@ module DTR
     end
 
     def broadcast_list=(list)
-      require 'dtr/shared'
       DTR.configuration.broadcast_list = list
       DTR.configuration.save
     end
 
     def agent_listen_port=(port)
-      require 'dtr/shared'
       DTR.configuration.agent_listen_port = port
       DTR.configuration.save
     end
 
     def group=(group)
-      require 'dtr/shared'
       DTR.configuration.group = group
       DTR.configuration.save
     end
 
+    def agent_env_setup_cmd=(cmd)
+      DTR.configuration.agent_env_setup_cmd = cmd
+      DTR.configuration.save
+    end
+
+    def agent_runners=(runners)
+      DTR.configuration.agent_runners = runners
+      DTR.configuration.save
+    end
+
+    def agent_runners
+      DTR.configuration.agent_runners
+    end
+
     def monitor
       require 'dtr/monitor'
       DTR.logger('dtr_monitor.log')
       Monitor.new.start
     end
 
-    # For safe fork & kill sub process, should use Process.kill and Process.fork
-    # At least have problem on ruby 1.8.6 114 with Kernel#kill & fork
-    def kill_process(pid)
-      Process.kill 'TERM', pid rescue nil
-    end
-
-    def fork_process(&block)
-      Process.fork(&block)
+    DTR_CMD = File.expand_path(File.dirname(__FILE__) + '/../../bin/dtr')
+    def run_script(cmd)
+      dtr = (RUBY_PLATFORM =~ /java/ || ENV['JRUBY_DTR']) ? "jruby -S #{DTR_CMD.inspect}" : "ruby #{DTR_CMD.inspect}"
+      system "#{dtr} -e #{cmd.inspect}"
     end
   end
 end