xli-dtr 1.0.0 → 1.1.0

data/lib/dtr/agent/sync_codebase.rb

@@ -17,27 +17,18 @@ module DTR
     module WorkingEnvExt
       include SyncService
 
-      def self.included(base)
-        base.alias_method_chain :setup_env, :sync_codebase
-        base.alias_method_chain :working_dir, :sync_codebase
-      end
-
-      def setup_env_with_sync_codebase(setup_env_cmd)
-        unless same_working_dir_with_master_process?
-          Dir.chdir(working_dir_without_sync_codebase) do
-            sync_codebase
+      def synchronize_for(runners)
+        Dir.chdir(base_dir) do
+          sync_codebase do
+            FileUtils.rm_rf(package_name)
+            do_work(unpackage_cmd)
+            runners.each do |runner_name|
+              dir = File.expand_path escape_dir(runner_name)
+              FileUtils.rm_rf(dir)
+              FileUtils.cp_r(package_name, dir)
+            end
           end
         end
-        setup_env_without_sync_codebase(setup_env_cmd)
-      end
-
-      def working_dir_with_sync_codebase
-        same_working_dir_with_master_process? ? Dir.pwd : File.join(working_dir_without_sync_codebase, package_name)
-      end
-
-      private
-      def same_working_dir_with_master_process?
-        self[:host] == Socket.gethostname && self[:pwd] == Dir.pwd
       end
     end
   end

data/lib/dtr/agent/sync_logger.rb

@@ -61,6 +61,7 @@ module DTR
       private
       def with_decorating_message(level, msg, &block)
         raise 'Should not use block to send log remotely' if block_given?
+        msg = "#{ENV['DTR_RUNNER_NAME']}: #{msg}"if ENV['DTR_RUNNER_NAME']
         @logger.send(level, decorate_message(msg))
       end
     end

data/lib/dtr/agent/worker.rb

@@ -15,74 +15,26 @@
 module DTR
 
   module Agent
-    # Worker works during one dtr test task running.
-    # Worker manages Herald & Runner processes life cycle.
+    # Worker watchs runner processes.
     class Worker
-      def initialize(runner_names, agent_env_setup_cmd)
-        @runner_names = runner_names.is_a?(Array) ? runner_names : [runner_names.to_s]
-        @agent_env_setup_cmd = agent_env_setup_cmd
-        @runner_pids = []
-        @herald = nil
-        @working_env_key = :working_env
-        @env_store = EnvStore.new
+      def initialize
+        @runners_group = ThreadGroup.new
       end
 
-      def launch
-        DTR.info {"=> Agent worker started at: #{Dir.pwd}, pid: #{Process.pid}"}
-        setup
-        begin
-          run
-        ensure
-          teardown
-          DTR.info {"Agent worker is dieing"}
+      def watch_runners
+        DTR.configuration.agent_runners.each do |name|
+          runner_thread = Thread.start { DTR.run_script("DTR::Agent::Runner.new(#{name.inspect}).start") }
+          runner_thread[:runner_name] = name
+          @runners_group.add runner_thread
         end
-      end
-
-      private
-      def setup
-        @env_store[@working_env_key] = nil
-      end
-
-      def teardown
-        unless @runner_pids.blank?
-          @runner_pids.each{ |pid| DTR.kill_process pid }
-          DTR.info {"=> All runners(#{@runner_pids.join(", ")}) were killed." }
-          @runner_pids = []
-        end
-        if @herald
-          DTR.kill_process @herald
-          @herald = nil
-          DTR.info {"=> Herald is killed."}
-        end
-      end
-
-      def run
-        herald
-        runners
-        DTR.info {"=> All agent worker sub processes exited."}
-      end
-
-      def herald
-        @herald = DTR.fork_process { Herald.new @working_env_key, @agent_env_setup_cmd, @runner_names }
-        Process.waitpid @herald
-        exit(-1) unless $?.exitstatus == 0
-      end
 
-      def runners
-        working_env = @env_store[@working_env_key]
+        yield
 
-        @runner_names.each do |name|
-          @runner_pids << DTR.fork_process {
-            at_exit {
-              # exit anyway, for DRb may hang on the process to be a deadwalk
-              exit!
-            }
-            working_env.within do
-              Runner.start name, working_env
-            end
-          }
+        while @runners_group.list.length > 0
+          alive_runners = @runners_group.list.collect {|r| r[:runner_name]}
+          DTR.info { "Waiting for #{alive_runners.join(', ').downcase} shutdown" }
+          sleep 1
         end
-        Process.waitall
       end
     end
   end

data/lib/dtr/agent/working_env_ext.rb

@@ -14,33 +14,67 @@
 
 module DTR
   module Agent
-    # Provides working environment setup beheaviours for Herald & Runner process
     module WorkingEnvExt
-      def within
-        ENV['DTR_MASTER_ENV'] = self[:dtr_master_env]
+      def base_dir
+        return @base_dir if defined?(@base_dir)
+        project_specific_len = 20
+        project_name = self[:pwd].length > project_specific_len ? self[:pwd][-project_specific_len..-1] : self[:pwd]
+        @base_dir = File.expand_path FileUtils.mkdir_p(File.join(escape_dir(self[:host]), escape_dir(project_name)))
+      end
+
+      def load_environment(&block)
+        working_dir = FileUtils.mkdir_p(File.join(base_dir, escape_dir(ENV['DTR_RUNNER_NAME'])))
         Dir.chdir(working_dir) do
-          yield
+          DTR.info "Loading environment at #{Dir.pwd}"
+
+          ENV['DTR_MASTER_ENV'] = dtr_master_env
+
+          setup_environment
+
+          append_libs_to_load_path
+          require_files
+
+          block.call
         end
       end
 
-      def setup_env(setup_env_cmd)
-        within do
-          Cmd.execute(setup_env_cmd || self[:agent_env_setup_cmd])
+      private
+
+      def setup_environment_command
+        DTR.configuration.agent_env_setup_cmd || self.agent_env_setup_cmd
+      end
+
+      def setup_environment
+        unless Cmd.execute(setup_environment_command)
+          raise "Stopped for setup working environment failed."
         end
       end
 
-      private
-      def escape(str)
-        str.gsub(/[^a-zA-Z0-9]/, '_')
+      def escape_dir(str)
+        str.to_s.gsub(/[^a-zA-Z0-9]/, '_')
       end
 
-      def working_dir
-        return @working_dir if defined?(@working_dir)
-        project_specific_len = 20
-        project_name = self[:pwd].length > project_specific_len ? self[:pwd][-project_specific_len..-1] : self[:pwd]
-        @working_dir = File.join(escape(self[:host]), escape(project_name))
-        FileUtils.mkdir_p(@working_dir)
-        @working_dir
+      def append_libs_to_load_path
+        libs.select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
+          $LOAD_PATH << lib
+          DTR.debug {"appended lib: #{lib}"}
+        end
+        DTR.info {"libs appended"}
+        DTR.debug {"$LOAD_PATH: #{$LOAD_PATH.inspect}"}
+      end
+
+      def require_files
+        files.each do |f|
+          begin
+            #use require instead of load to avoid load file twice.
+            require f unless f =~ /^-/
+            DTR.debug {"required #{f}"}
+          rescue LoadError => e
+            DTR.error {"No such file to load -- #{f}"}
+            DTR.debug {"Environment: #{self}"}
+          end
+        end
+        DTR.info {"test files loaded"}
       end
     end
   end

data/lib/dtr/agent/working_status.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    module WorkingStatus
+      WORKING_STATUS_KEY = :runners_working_status
+      def runners_should_be_working
+        @store[WORKING_STATUS_KEY] = Time.now
+      end
+
+      def runners_should_be_working?
+        if time = @store[WORKING_STATUS_KEY]
+          (time - Time.now) <= follower_listen_heartbeat_timeout
+        end
+      end
+
+      def agent_is_going_to_sleep
+        @store[WORKING_STATUS_KEY] = nil
+      end
+    end
+  end
+end

data/lib/dtr/agent.rb

@@ -15,8 +15,11 @@
 require 'dtr/shared'
 require 'dtr/agent/working_env_ext'
 require 'dtr/agent/sync_codebase'
+require 'dtr/agent/rails_ext'
+require 'dtr/agent/working_status'
 
 require 'dtr/agent/sync_logger'
+require 'dtr/agent/process_root'
 require 'dtr/agent/brain'
 require 'dtr/agent/worker'
 require 'dtr/agent/test_unit'
@@ -24,15 +27,19 @@ require 'dtr/agent/herald'
 require 'dtr/agent/test_case'
 require 'dtr/agent/runner'
 
+DTR.logger('dtr_agent.log')
+
 module DTR
   module Agent
-    def start(runner_names=["Distributed Test Runner"], agent_env_setup_cmd=nil)
-      DTR.logger('dtr_agent.log')
-      Brain.new(runner_names, agent_env_setup_cmd).hypnotize
+    def start(action=:hypnotize)
+      Brain.new.send(action)
     end
     
     module_function :start
+
+    WorkingEnv.send(:include, WorkingEnvExt)
+    WorkingEnv.send(:include, SyncCodebase::WorkingEnvExt)
+    WorkingEnv.send(:include, RailsExt::WorkingEnvExt)
+    Configuration.send(:include, WorkingStatus)
   end
-  WorkingEnv.send(:include, Agent::WorkingEnvExt)
-  WorkingEnv.send(:include, SyncCodebase::WorkingEnvExt)
 end

data/lib/dtr/facade.rb

@@ -15,13 +15,8 @@
 module DTR
   module Facade
     def start_agent
-      launch_agent(DTR_AGENT_OPTIONS[:runners], DTR_AGENT_OPTIONS[:agent_env_setup_cmd])
-    end
-
-    def launch_agent(names, setup=nil)
       require 'dtr/agent'
-      names = names || "DTR(#{Time.now})"
-      DTR::Agent.start(names, setup)
+      DTR::Agent.start
     end
 
     def lib_path
@@ -29,37 +24,44 @@ module DTR
     end
 
     def broadcast_list=(list)
-      require 'dtr/shared'
       DTR.configuration.broadcast_list = list
       DTR.configuration.save
     end
 
     def agent_listen_port=(port)
-      require 'dtr/shared'
       DTR.configuration.agent_listen_port = port
       DTR.configuration.save
     end
 
     def group=(group)
-      require 'dtr/shared'
       DTR.configuration.group = group
       DTR.configuration.save
     end
 
+    def agent_env_setup_cmd=(cmd)
+      DTR.configuration.agent_env_setup_cmd = cmd
+      DTR.configuration.save
+    end
+
+    def agent_runners=(runners)
+      DTR.configuration.agent_runners = runners
+      DTR.configuration.save
+    end
+
+    def agent_runners
+      DTR.configuration.agent_runners
+    end
+
     def monitor
       require 'dtr/monitor'
       DTR.logger('dtr_monitor.log')
       Monitor.new.start
     end
 
-    # For safe fork & kill sub process, should use Process.kill and Process.fork
-    # At least have problem on ruby 1.8.6 114 with Kernel#kill & fork
-    def kill_process(pid)
-      Process.kill 'TERM', pid rescue nil
-    end
-
-    def fork_process(&block)
-      Process.fork(&block)
+    DTR_CMD = File.expand_path(File.dirname(__FILE__) + '/../../bin/dtr')
+    def run_script(cmd)
+      dtr = (RUBY_PLATFORM =~ /java/ || ENV['JRUBY_DTR']) ? "jruby -S #{DTR_CMD.inspect}" : "ruby #{DTR_CMD.inspect}"
+      system "#{dtr} -e #{cmd.inspect}"
     end
   end
 end

data/lib/dtr/master.rb

@@ -24,7 +24,9 @@ module DTR
       DTR.info {""}
       DTR.info {"--------------------beautiful line--------------------------"}
       DTR.info {"Master process started at #{Time.now}"}
+      DTR.info {"Master heartbeat interval: #{DTR.configuration.master_heartbeat_interval}"}
 
+      DTR.root = Dir.pwd
       DTR.configuration.with_rinda_server do
         provide_working_env WorkingEnv.new
         with_wakeup_agents(&block)

data/lib/dtr/monitor.rb

@@ -23,7 +23,7 @@ module DTR
       include Adapter::Master
 
       def start
-        DTR.fork_process do
+        Process.fork do
           monitor
         end
       end
@@ -44,7 +44,7 @@ module DTR
       include Service::Agent
 
       def start
-        DTR.fork_process do
+        Process.fork do
           begin
             loop do
               monitor

data/lib/dtr/raketasks.rb

@@ -18,6 +18,7 @@ require 'rake/testtask'
 require 'rake/tasklib'
 
 require 'dtr'
+require 'dtr/shared/root'
 require 'dtr/shared/ruby_ext'
 require 'dtr/shared/utils'
 require 'dtr/shared/sync_codebase/package'
@@ -85,23 +86,17 @@ module DTR
 
       desc "Run tests with DTR injected"
       task @name do
-        @agent = start_agent
+        start_agent
         run_code = ''
-        begin
-          RakeFileUtils.verbose(@verbose) do
-            run_code = rake_loader
-            @ruby_opts.unshift( "-I#{lib_path}" )
-            @ruby_opts.unshift( "-w" ) if @warning
-            
-            ruby @ruby_opts.join(" ") +
-              " \"#{run_code}\" " +
-              file_list.unshift('dtr/test_unit_injection.rb').collect { |fn| "\"#{fn}\"" }.join(' ') +
-              " #{option_list}"
-          end
-        ensure
-          if defined?(@agent)
-            DTR.kill_process @agent
-          end
+        RakeFileUtils.verbose(@verbose) do
+          run_code = rake_loader
+          @ruby_opts.unshift( "-I#{lib_path}" )
+          @ruby_opts.unshift( "-w" ) if @warning
+          
+          ruby @ruby_opts.join(" ") +
+            " \"#{run_code}\" " +
+            file_list.unshift('dtr/test_unit_injection.rb').collect { |fn| "\"#{fn}\"" }.join(' ') +
+            " #{option_list}"
         end
       end
       self
@@ -113,9 +108,9 @@ module DTR
       runner_names = []
       self.processes.to_i.times {|i| runner_names << "runner#{i}"}
       
-      DTR.fork_process do
-        DTR_AGENT_OPTIONS[:runners] = runner_names if DTR_AGENT_OPTIONS[:runners].empty?
-        DTR.start_agent
+      DTR.agent_runners = runner_names if DTR.agent_runners.blank?
+      Thread.start do
+        DTR.run_script('DTR::Agent.start(:hypnotize_once)')
       end
     end
   end

data/lib/dtr/shared/adapter/base.rb

@@ -0,0 +1,118 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+
+module DTR
+  module Adapter
+    WAKEUP_MESSAGE = 'wakeup'
+    SLEEP_MESSAGE = 'sleep'
+
+    module Follower
+      def wakeup?
+        msg, host, group = listen
+        if group == DTR.configuration.group && msg == Adapter::WAKEUP_MESSAGE
+          ip, port = host.split(':')
+          DTR.configuration.rinda_server_port = port.to_i
+          DTR.configuration.broadcast_list = [ip]
+          DTR.configuration.save
+          DTR.info {"=> Rinda server port: #{DTR.configuration.rinda_server_port}"}
+          DTR.info {"=> Broadcast list: #{DTR.configuration.broadcast_list.inspect}"}
+          @wakeup_for_host = host
+          true
+        end
+      end
+
+      def sleep?
+        return true unless defined?(@wakeup_for_host)
+        Timeout.timeout(DTR.configuration.follower_listen_heartbeat_timeout) do
+          until (msg, host = listen) && host == @wakeup_for_host; end
+          msg == Adapter::SLEEP_MESSAGE
+        end
+      rescue Timeout::Error => e
+        DTR.info {"Timeout while listening command"}
+        true
+      end
+
+      def relax
+        if defined?(@soc) && @soc
+          @soc.close rescue nil
+        end
+      end
+
+      private
+      def listen
+        unless defined?(@soc)
+          @soc = UDPSocket.open
+          @soc.bind('', DTR.configuration.agent_listen_port)
+          DTR.info("DTR Agent is listening on port #{DTR.configuration.agent_listen_port}")
+        end
+        message, client_address = @soc.recvfrom(400)
+        cmd, port, group = message.split
+
+        hostname = client_address[2]
+        host_ip = client_address[3]
+        DTR.info {"Received: #{cmd} for group #{group} from #{hostname}(#{host_ip}):#{port}"}
+        [cmd, "#{host_ip}:#{port}", group]
+      end
+    end
+
+    module Master
+      def hypnotize_agents
+        yell_agents("#{Adapter::SLEEP_MESSAGE} #{DTR.configuration.rinda_server_port}")
+      end
+
+      def with_wakeup_agents(&block)
+        heartbeat = Thread.new do
+          loop do
+            do_wakeup_agents
+            sleep(DTR.configuration.master_heartbeat_interval)
+          end
+        end
+        #heartbeat thread should have high priority for agent is listening
+        heartbeat.priority = Thread.current.priority + 10
+        block.call
+      ensure
+        #kill heartbeat first, so that agents wouldn't be wakeup after hypnotized
+        Thread.kill heartbeat rescue nil
+        hypnotize_agents rescue nil
+      end
+
+      def do_wakeup_agents
+        yell_agents("#{Adapter::WAKEUP_MESSAGE} #{DTR.configuration.rinda_server_port} #{DTR.configuration.group}")
+      end
+
+      private
+      def yell_agents(msg)
+        DTR.info {"yell agents #{msg}: #{DTR.configuration.broadcast_list.inspect}"}
+        DTR.configuration.broadcast_list.each do |it|
+          broadcast(it, msg)
+        end
+      end
+
+      def broadcast(it, msg)
+        soc = UDPSocket.open
+        begin
+          soc.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, true)
+          DTR.debug {"broadcast sending #{msg} to #{it}"}
+          soc.send(msg, 0, it, DTR.configuration.agent_listen_port)
+        rescue
+          nil
+        ensure
+          soc.close
+        end
+      end
+    end
+  end
+end

data/lib/dtr/shared/adapter/mortality.rb

@@ -0,0 +1,41 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+
+module DTR
+  module Adapter
+    DEAD_MESSAGE = 'die'
+
+    module Follower
+      private
+
+      def listen_with_dead_message
+        cmd, host, group = listen_without_dead_message
+        if cmd == Adapter::DEAD_MESSAGE && group == DTR.configuration.group
+          DTR.info{"Master need me die. So I am dieing."}
+          raise Interrupt, "Master need me die."
+        end
+        [cmd, host, group]
+      end
+      alias_method_chain :listen, :dead_message
+    end
+
+    module Master
+      def group_agents_should_die(group = DTR.configuration.group)
+        yell_agents("#{Adapter::DEAD_MESSAGE} #{DTR.configuration.rinda_server_port} #{group}")
+      end
+    end
+  end
+end