xli-dtr 0.0.4 → 0.0.5

data/lib/dtr/agent/herald.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+
+  module Agent
+    class Herald
+      include Service::WorkingEnv
+      include Service::Agent
+
+      def initialize(working_env_key, agent_env_setup_cmd, runners)
+        @working_env_key = working_env_key
+        @agent_env_setup_cmd = agent_env_setup_cmd
+        @runners = runners
+        @env_store = EnvStore.new
+        start_off
+      end
+
+      def start_off
+        start_service
+        DTR.info "=> Herald starts off..."
+        provide_agent_info(@agent_env_setup_cmd, @runners)
+
+        working_env = lookup_working_env
+        DTR.info "=> Got working environment created at #{working_env[:created_at]} by #{working_env[:host]}"
+        if working_env[:files].blank?
+          DTR.error "No test files need to load?(working env: #{working_env})"
+          return
+        end
+
+        if working_env.setup_env(@agent_env_setup_cmd)
+          @env_store[@working_env_key] = working_env
+        else
+          DTR.info {'Setup working environment failed, no runner started.'}
+        end
+      end
+    end
+  end
+end

data/lib/dtr/agent/runner.rb

@@ -0,0 +1,100 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    class Runner
+      include DRbUndumped
+      include Service::Runner
+
+      def self.start(name, env)
+        self.new(name, env).start
+        DTR.info "=> Runner #{name} provided"
+        DRb.thread.join if DRb.thread
+      end
+
+      attr_reader :name, :identifier
+
+      def initialize(name, env)
+        @name = name
+        @identifier = env[:identifier]
+        @env = env
+        @started = 0
+        @run_finished = 0
+      end
+      
+      def start
+        #start service first, so that all logs can be sync with master process
+        start_service
+        DTR.info("=> Starting runner #{name} at #{Dir.pwd}, pid: #{Process.pid}")
+        init_environment
+        provide_runner(self)
+      end
+
+      def init_environment
+        DTR.info "#{name}: Initialize working environment..."
+        @env[:libs].select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
+          $LOAD_PATH << lib
+          DTR.debug "#{name}: appended lib: #{lib}"
+        end
+        DTR.info "#{name}: libs loaded"
+        DTR.debug "#{name}: $LOAD_PATH: #{$LOAD_PATH.inspect}"
+
+        @env[:files].each do |f|
+          begin
+            load f unless f =~ /^-/
+            DTR.debug "#{name}: loaded #{f}"
+          rescue LoadError => e
+            DTR.error "#{name}: No such file to load -- #{f}"
+            DTR.debug "Environment: #{@env}"
+          end
+        end
+        DTR.info "#{name}: test files loaded"
+      end
+
+      def run(test, result, &progress_block)
+        DTR.info "#{name}: +"
+        DTR.debug "#{name}: running #{test}..."
+        @started += 1
+        test.run(result, &progress_block)
+      rescue DRb::DRbConnError => e
+        DTR.info "Rescued DRb::DRbConnError(#{e.message}), while running test: #{test.name}. The master process may be stopped."
+      rescue Exception => e
+        DTR.error "Unexpected exception: #{e.message}"
+        DTR.error e.backtrace.join("\n")
+        result.add_error(Test::Unit::Error.new(test.name, e))
+        result.add_run
+        progress_block.call(Test::Unit::TestCase::FINISHED, test.name)
+      ensure
+        DTR.debug "#{name}: done #{test}"
+        @run_finished += 1
+        provide_runner(self)
+      end
+
+      def reboot
+        DTR.info "#{self} is rebooting. Ran #{@started} tests, finished #{@run_finished}."
+        provide_runner(self)
+      end
+
+      def shutdown
+        DTR.info "#{self} is shutting down. Ran #{@started} tests, finished #{@run_finished}."
+        stop_service rescue exit!
+      end
+
+      def to_s
+        "Runner #{@name}"
+      end
+    end
+  end
+end

data/lib/dtr/agent/sync_codebase.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncCodebase
+    module WorkingEnvExt
+      include SyncService
+
+      def self.included(base)
+        base.alias_method_chain :setup_env, :sync_codebase
+        base.alias_method_chain :working_dir, :sync_codebase
+      end
+
+      def setup_env_with_sync_codebase(setup_env_cmd)
+        unless same_working_dir_with_master_process?
+          Dir.chdir(working_dir_without_sync_codebase) do
+            sync_codebase
+          end
+        end
+        setup_env_without_sync_codebase(setup_env_cmd)
+      end
+
+      def working_dir_with_sync_codebase
+        same_working_dir_with_master_process? ? Dir.pwd : File.join(working_dir_without_sync_codebase, package_name)
+      end
+
+      private
+      def same_working_dir_with_master_process?
+        self[:host] == Socket.gethostname && self[:pwd] == Dir.pwd
+      end
+    end
+  end
+end

data/lib/dtr/agent/sync_logger.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncLogger
+
+    # Synchronizer loads SyncLogger from master process
+    # Any process need sync logs with master process should start service first.
+    # For logs would be sent back to master process, output log code should use 
+    # message string instead of block, for example: 
+    #    right: DTR.debug 'message'
+    #    not:   DTR.debug { 'message' }
+    module Synchronizer
+      def self.included(base)
+        base.alias_method_chain :start_service, :sync_logger
+      end
+
+      def start_service_with_sync_logger
+        start_service_without_sync_logger
+        if logger_tuple = lookup_ring.read_all([:logger, nil]).first
+          sync_logger = logger_tuple[1]
+          DTR.logger = sync_logger
+        end
+      end
+    end
+  end
+
+  Service::Rinda.send(:include, SyncLogger::Synchronizer)
+end

data/lib/dtr/agent/test_unit.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test/unit'
+require 'test/unit/testcase'
+
+module DTR
+  module Agent
+    module TestCaseExt
+      include MessageDecorator
+      def self.included(base)
+        base.alias_method_chain :add_failure, :decorating_source
+        base.alias_method_chain :add_error, :decorating_source
+      end
+
+      def add_error_with_decorating_source(exception)
+        add_error_without_decorating_source(DTR::RemoteError.new(exception))
+      end
+
+      def add_failure_with_decorating_source(message, all_locations=caller())
+        add_failure_without_decorating_source(decorate_message(message, 'Assertion failure'), all_locations)
+      end
+    end
+  end
+end
+
+Test::Unit.run = true
+
+class Test::Unit::TestCase
+  include DTR::Agent::TestCaseExt
+end

data/lib/dtr/agent/worker.rb

@@ -0,0 +1,92 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+
+  module Agent
+    class Worker
+      def initialize(runner_names, agent_env_setup_cmd)
+        @runner_names = runner_names.is_a?(Array) ? runner_names : [runner_names.to_s]
+        @agent_env_setup_cmd = agent_env_setup_cmd
+        @runner_pids = []
+        @herald = nil
+        @working_env_key = :working_env
+        @env_store = EnvStore.new
+      end
+
+      def launch
+        DTR.info "=> Agent worker started at: #{Dir.pwd}, pid: #{Process.pid}"
+        setup
+        begin
+          run
+        ensure
+          teardown
+          DTR.info "Agent worker is dieing"
+        end
+      end
+
+      private
+      def setup
+        @env_store[@working_env_key] = nil
+      end
+
+      def teardown
+        kill_all_runners
+        if @herald
+          Process.kill 'TERM', @herald rescue nil
+          @herald = nil
+          DTR.info "=> Herald is killed."
+        end
+      end
+
+      def run
+        @herald = drb_fork { Herald.new @working_env_key, @agent_env_setup_cmd, @runner_names }
+        while @env_store[@working_env_key].nil?
+          sleep(1)
+        end
+        working_env = @env_store[@working_env_key]
+
+        @runner_names.each do |name|
+          @runner_pids << drb_fork {
+            working_env.within do
+              Runner.start name, working_env
+            end
+          }
+        end
+        Process.waitall
+        DTR.info "=> All agent worker sub processes exited."
+      end
+
+      def kill_all_runners
+        unless @runner_pids.blank?
+          @runner_pids.each{ |pid| Process.kill 'TERM', pid rescue nil }
+          DTR.info "=> All runners(#{@runner_pids.join(", ")}) were killed." 
+          @runner_pids = []
+        end
+      end
+
+      def drb_fork
+        Process.fork do
+          begin
+            yield
+          rescue Interrupt, SystemExit, SignalException
+          rescue Exception => e
+            DTR.error "Worker drb fork is stopped by Exception => #{e.class.name}, message => #{e.message}"
+            DTR.info e.backtrace.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dtr/agent/working_env_ext.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    module WorkingEnvExt
+      def within
+        ENV['DTR_MASTER_ENV'] = self[:dtr_master_env]
+        Dir.chdir(working_dir) do
+          yield
+        end
+      end
+
+      def setup_env(setup_env_cmd)
+        within do
+          Cmd.execute(setup_env_cmd || self[:agent_env_setup_cmd])
+        end
+      end
+
+      private
+      def escape(str)
+        str.gsub(/[^a-zA-Z0-9]/, '_')
+      end
+
+      def working_dir
+        return @working_dir if defined?(@working_dir)
+        project_name = self[:pwd].length > 20 ? self[:pwd][-20..-1] : self[:pwd]
+        @working_dir = File.join(escape(self[:host]), escape(project_name))
+        FileUtils.mkdir_p(@working_dir)
+        @working_dir
+      end
+    end
+  end
+end

data/lib/dtr/agent.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared'
+require 'dtr/agent/working_env_ext'
+require 'dtr/agent/sync_codebase'
+
+require 'dtr/agent/sync_logger'
+require 'dtr/agent/brain'
+require 'dtr/agent/worker'
+require 'dtr/agent/test_unit'
+require 'dtr/agent/herald'
+require 'dtr/agent/runner'
+
+module DTR
+  module Agent
+    def start(runner_names=["Distributed Test Runner"], agent_env_setup_cmd=nil)
+      DTR.logger('dtr_agent.log')
+      Brain.new(runner_names, agent_env_setup_cmd).hypnotize
+    end
+    
+    module_function :start
+  end
+  WorkingEnv.send(:include, Agent::WorkingEnvExt)
+  WorkingEnv.send(:include, SyncCodebase::WorkingEnvExt)
+end

data/lib/dtr/master.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared'
+
+module DTR
+  module Master
+    def with_dtr_master(&block)
+      if defined?(ActiveRecord::Base)
+        ActiveRecord::Base.clear_active_connections! rescue nil
+      end
+
+      DTR.info ""
+      DTR.info "--------------------beautiful line--------------------------"
+      DTR.info {"Master process started at #{Time.now}"}
+
+      DTR.configuration.with_rinda_server do
+        provide_working_env WorkingEnv.new
+        with_wakeup_agents(&block)
+      end
+    end
+
+    include Adapter::Master
+    include Service::WorkingEnv
+    include SyncCodebase::MasterExt
+  end
+
+  Configuration.send(:include, SyncLogger::Provider)
+end

data/lib/dtr/monitor.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/master'
+
+module DTR
+  class Monitor
+    include Adapter::Master
+    include Service::Agent
+    def start
+      DTR.configuration.with_rinda_server do
+        monitor_thread = Thread.new do
+          new_agent_monitor.each { |t| puts t.last.last }
+        end
+        puts "Monitor process started at #{Time.now}"
+      
+        with_wakeup_agents do
+          begin
+            monitor_thread.join
+          rescue Interrupt
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dtr/raketasks.rb

@@ -14,11 +14,15 @@
 
 require "rubygems"
 require 'dtr'
+require 'dtr/shared/sync_codebase/package'
+require 'dtr/shared/ruby_ext'
+require 'rake'
 require 'rake/testtask'
+require 'rake/tasklib'
 
 module DTR
-  class MPTask < Rake::TestTask
-    attr_accessor :processes, :runner_options, :start_server
+  class TestTask < Rake::TestTask
+    attr_accessor :processes
     
     def define
       @libs.unshift DTR.lib_path
@@ -26,8 +30,7 @@ module DTR
 
       desc "Run tests" + (@name==:test ? "" : " for #{@name}")
       task @name do
-        DTR.start_server_daemon_mode if start_server?
-        start_runners
+        @agent = start_agent
         run_code = ''
         begin
           RakeFileUtils.verbose(@verbose) do
@@ -41,9 +44,8 @@ module DTR
               " #{option_list}"
           end
         ensure
-          DTR.stop_runners_daemon_mode rescue nil
-          if start_server?
-            DTR.stop_server_daemon_mode rescue nil
+          if defined?(@agent)
+            Process.kill 'TERM', @agent rescue nil
           end
         end
       end
@@ -54,16 +56,75 @@ module DTR
       @processes ? @processes.to_i : 2
     end
     
-    def start_server?
-      defined?(@start_server) ? @start_server : true
-    end
-    
     private
-    def start_runners
+    def start_agent
       return if self.processes.to_i <= 0
       runner_names = []
       self.processes.to_i.times {|i| runner_names << "runner#{i}"}
-      %x[dtr -r #{runner_names.join(',')} -D #{runner_options}]
+      
+      Process.fork do
+        DTR_AGENT_OPTIONS[:runners] = runner_names if DTR_AGENT_OPTIONS[:runners].empty?
+        DTR.start_agent
+      end
+    end
+  end
+
+  # The following task is copied & modified from 'rake/packagetask'
+  class PackageTask < Rake::TaskLib
+    include SyncCodebase::Package
+    # List of files to be included in the package.
+    attr_accessor :package_files
+
+    # Tar command for gzipped or bzip2ed archives.  The default is 'tar'.
+    attr_accessor :tar_command
+
+    # Create a Package Task with the given name and version. 
+    def initialize
+      @package_files = Rake::FileList.new
+      @tar_command = 'tar'
+      yield self if block_given?
+      define
+    end
+
+    # Create the tasks defined by this task library.
+    def define
+      desc "Build packages for dtr task"
+      task :dtr_package
+
+      desc "Force a rebuild of the package files for dtr task"
+      task :dtr_repackage => [:dtr_clobber_package, :dtr_package]
+
+      desc "Remove package for dtr task" 
+      task :dtr_clobber_package do
+        rm_r package_dir rescue nil
+      end
+
+      file, flag = package_file, 'j'
+      task :dtr_package => ["#{package_dir}/#{file}"]
+
+      file "#{package_dir}/#{file}" => [package_dir_path] do
+        chdir(package_dir) do
+          sh %{#{@tar_command} #{flag}cvf #{file} #{package_name}}
+        end
+      end
+
+      directory package_dir
+
+      file package_dir_path do
+        mkdir_p package_dir rescue nil
+        @package_files.each do |fn|
+          f = File.join(package_dir_path, fn)
+          fdir = File.dirname(f)
+          mkdir_p(fdir) if !File.exist?(fdir)
+          if File.directory?(fn)
+            mkdir_p(f)
+          else
+            rm_f f
+            safe_ln(fn, f)
+          end
+        end
+      end
+      self
     end
   end
 end