xli-dtr 0.0.4 → 0.0.5

data/lib/dtr/shared/sync_codebase/sync_service.rb

@@ -0,0 +1,41 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncCodebase
+    module SyncService
+      include Package
+      include Service::File
+      def sync_codebase
+        DTR.info("start sync codebase, clean #{File.join(Dir.pwd, package_name)}")
+        FileUtils.rm_rf(File.join(Dir.pwd, package_name))
+
+        DTR.info("lookup codebase file")
+        codebase = lookup_file
+        DTR.info("receiving codebase: #{package_copy_file}")
+        File.open(package_copy_file, 'w') do |f|
+          codebase.write(f)
+        end
+        unless File.exists?(package_copy_file)
+          raise "#{package_copy_file} does not exist, sync codebase failed."
+        end
+        unless Cmd.execute("tar -xjf #{package_copy_file}")
+          raise "Extracting #{package_copy_file} by 'tar' failed."
+        end
+        DTR.info("sync codebase finished, clean #{package_copy_file}")
+        FileUtils.rm_f(package_copy_file)
+      end
+    end
+  end
+end

data/lib/dtr/shared/sync_codebase.rb

@@ -0,0 +1,18 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared/sync_codebase/package'
+require 'dtr/shared/sync_codebase/codebase'
+require 'dtr/shared/sync_codebase/sync_service'
+require 'dtr/shared/sync_codebase/master_ext'

data/lib/dtr/shared/sync_logger.rb

@@ -0,0 +1,72 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'delegate'
+
+module DTR
+  module SyncLogger
+    class UndumpedLogger
+      include DRbUndumped
+      include MessageDecorator
+
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def debug(message=nil, &block)
+        with_decorating_message(:debug, message, &block)
+      end
+
+      def info(message=nil, &block)
+        with_decorating_message(:info, message, &block)
+      end
+
+      def error(message=nil, &block)
+        with_decorating_message(:error, message, &block)
+      end
+
+      def datetime_format=(format)
+        @logger.datetime_format = format
+      end
+
+      def level=(level)
+        @logger.level = level
+      end
+
+      private
+      def with_decorating_message(level, msg, &block)
+        if block_given?
+          @logger.send(level) do
+            decorate_message(block.call)
+          end
+        else
+          @logger.send(level, decorate_message(msg))
+        end
+      end
+    end
+
+    module Provider
+      def self.included(base)
+        base.alias_method_chain :with_rinda_server, :providing_sync_logger
+      end
+
+      def with_rinda_server_with_providing_sync_logger(&block)
+        with_rinda_server_without_providing_sync_logger do
+          lookup_ring.write [:logger, UndumpedLogger.new(DTR.logger)]
+          block.call
+        end
+      end
+    end
+  end
+end

data/lib/dtr/shared/utils/cmd.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  class CmdInterrupt < StandardError; end
+
+  class Cmd
+    def self.execute(cmd)
+      return true if cmd.nil? || cmd.empty?
+      DTR.info "Executing: #{cmd.inspect}"
+      output = %x[#{cmd} 2>&1]
+      # don't put the following message into a block which maybe passed to remote process
+      # and no $?  could be found
+      DTR.info "Execution is done, status: #{$?.exitstatus}"
+      DTR.error "#{cmd.inspect} output:\n#{output}" if $?.exitstatus != 0
+      $?.exitstatus == 0
+    end
+  end
+end

data/lib/dtr/shared/utils/env_store.rb

@@ -0,0 +1,60 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pstore'
+
+module DTR
+  class EnvStore
+    FILE_NAME = '.dtr_env_pstore' unless defined?(FILE_NAME)
+
+    def self.destroy
+      File.delete(FILE_NAME) if File.exist?(FILE_NAME)
+    end
+
+    def [](key)
+      return nil unless File.exist?(FILE_NAME)
+      
+      repository = PStore.new(FILE_NAME)
+      repository.transaction(true) do
+        repository[key]
+      end
+    end
+
+    def []=(key, value)
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        repository[key] = value
+      end
+    end
+    
+    def <<(key_value)
+      key, value = key_value
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        repository[key] = (repository[key] || []) << value
+      end
+    end
+    
+    def shift(key)
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        if array = repository[key]
+          array.shift
+          repository[key] = array
+        end
+      end
+    end
+  end
+
+end

data/lib/dtr/shared/utils/logger.rb

@@ -0,0 +1,71 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'logger'
+
+module DTR
+  module LoggerExt
+    def logger(file=nil)
+      @logger ||= create_default_logger(file)
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    def create_default_logger(file=nil)
+      dir = File.exist?('log') ? 'log' : '/tmp'
+      log_file = File.join(dir,  file || "dtr.log")
+      do_print "dtr logfile at #{log_file}\n"
+      logger = Logger.new(log_file, 1, 5*1024*1024)
+      logger.datetime_format = "%m-%d %H:%M:%S"
+      logger.level = (ENV['DTR_LOG_LEVEL'] || Logger::INFO).to_i
+      logger
+    end
+
+    def do_print(str)
+      unless ENV['DTR_ENV'] == 'test'
+        print str
+      end
+    end
+
+    def debug(message=nil, &block)
+      output(:debug, message, &block)
+    end
+
+    def info(message=nil, &block)
+      output(:info, message, &block)
+    end
+
+    def error(message=nil, &block)
+      output(:error, message, &block)
+    end
+
+    def output(level, msg=nil, &block)
+      logger.send(level) do
+        message = block_given? ? block.call : msg.to_s
+        # puts "log: #{message}"
+
+        #output message when it's an error for agent error log should be displayed in console
+        if level == :error
+          $stderr.puts ''
+          $stderr.puts "#{Time.now}: #{message}"
+        end
+        message
+      end
+    end
+  end
+
+  extend LoggerExt
+end

data/lib/dtr/shared/utils.rb

@@ -0,0 +1,17 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared/utils/logger'
+require 'dtr/shared/utils/env_store'
+require 'dtr/shared/utils/cmd'

data/lib/dtr/shared/working_env.rb

@@ -0,0 +1,38 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  class WorkingEnv
+    def initialize
+      files = (defined?($argv_dup) ? $argv_dup : []).dup
+      @env = {:libs => $LOAD_PATH.dup, :files => files, :created_at => Time.now.to_s, :dtr_master_env => ENV['DTR_MASTER_ENV'], :agent_env_setup_cmd => ENV['DTR_AGENT_ENV_SETUP_CMD'], :identifier => "#{Time.now.to_s}:#{rand}:#{object_id}", :host => Socket.gethostname, :pwd => Dir.pwd}
+    end
+
+    def [](key)
+      @env[key]
+    end
+    
+    def []=(key, value)
+      @env[key] = value
+    end
+    
+    def to_s
+      @env.inspect
+    end
+    
+    def ==(obj)
+      obj && obj[:identifier] == self[:identifier]
+    end
+  end
+end

data/lib/dtr/shared.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+require 'dtr/shared/ruby_ext'
+require 'dtr/shared/utils'
+require 'dtr/shared/message_decorator'
+require 'dtr/shared/working_env'
+require 'dtr/shared/service'
+require 'dtr/shared/configuration'
+require 'dtr/shared/sync_logger'
+require 'dtr/shared/adapter'
+require 'dtr/shared/sync_codebase'

data/lib/dtr/test_unit/drb_test_runner.rb

@@ -0,0 +1,57 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+
+module DTR
+  module TestUnit
+    class DRbTestRunner
+      include Service::Runner
+
+      def initialize(test, result, &progress_block)
+        @test = test
+        @result = result
+        @progress_block = progress_block
+      end
+
+      def run
+        if runner = lookup_runner
+          WorkerClub.instance.start_thread(self, runner)
+        else
+          self.run
+        end
+      end
+
+      def run_test_on(runner, timeout)
+        Timeout.timeout(timeout) do
+          runner.run(@test, @result, &@progress_block)
+        end
+      rescue Timeout::Error => e
+        DTR.info {"Run test timeout(#{timeout}), reboot runner"}
+        runner.reboot rescue nil
+        DTR.info {"rerun test: #{@test.name}"}
+        self.run
+      rescue DRb::DRbConnError => e
+        DTR.info {"DRb::DRbConnError(#{e.message}), rerun test: #{@test.name}"}
+        DTR.debug { e.backtrace.join("\n") }
+        self.run
+      rescue Exception => e
+        DTR.info{ "#{@test.name}, rescue an exception: #{e.message}, add error into result." }
+        @result.add_error(Test::Unit::Error.new(@test.name, e))
+        @result.add_run
+        @progress_block.call(Test::Unit::TestCase::FINISHED, @test.name)
+      end
+    end
+  end
+end

data/lib/dtr/test_unit/injection.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test/unit/testcase'
+require 'test/unit/ui/testrunnermediator'
+
+module DTR
+  def reject
+    return unless Test::Unit::UI::TestRunnerMediator.respond_to?(:reject_dtr)
+    Test::Unit::UI::TestRunnerMediator.reject_dtr
+  end
+
+  def inject
+    return if Test::Unit::UI::TestRunnerMediator.respond_to?(:reject_dtr)
+    Test::Unit::UI::TestRunnerMediator.send(:include, TestUnit::TestRunnerMediatorInjection)
+  end
+
+  module_function :reject, :inject
+end

data/lib/dtr/test_unit/test_case_injection.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module TestUnit
+    module TestCaseInjection
+
+      def self.included(base)
+        base.alias_method_chain :run, :dtr_injection
+        #have to use class_eval for adding it back after removed reject_dtr method
+        base.class_eval do
+          def self.reject_dtr
+            remove_method :run
+            alias_method :run, :run_without_dtr_injection
+            remove_method :run_without_dtr_injection
+            (class << self; self; end;).send :remove_method, :reject_dtr
+          end
+        end
+      end
+
+      def run_with_dtr_injection(result, &progress_block)
+        DRbTestRunner.new(self, result, &progress_block).run
+      end
+    end
+  end
+end

data/lib/dtr/test_unit/testrunnermediator_injection.rb

@@ -0,0 +1,72 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module TestUnit
+    module TestRunnerMediatorInjection
+
+      class SuiteWrapper
+        def initialize(suite)
+          @suite = suite
+        end
+        def run(result, &block)
+          @suite.run(result, &block)
+          WorkerClub.instance.graceful_shutdown
+        end
+        def size
+          @suite.size
+        end
+      end
+
+      def self.included(base)
+        base.send(:include, Master)
+        base.alias_method_chain :create_result, :thread_safe
+        base.alias_method_chain :run_suite, :dtr_injection
+        #have to use class_eval for adding it back after removed reject_dtr method
+        base.class_eval do
+          def self.reject_dtr
+            remove_method :run_suite
+            alias_method :run_suite, :run_suite_without_dtr_injection
+            remove_method :run_suite_without_dtr_injection
+
+            remove_method :create_result
+            alias_method :create_result, :create_result_without_thread_safe
+            remove_method :create_result_without_thread_safe
+
+            (class << self; self; end;).send :remove_method, :reject_dtr
+          end
+        end
+      end
+
+      def create_result_with_thread_safe
+        ThreadSafeTestResult.new(create_result_without_thread_safe)
+      end
+
+      def run_suite_with_dtr_injection
+        DTR.logger('dtr_master_process.log')
+        @suite = SuiteWrapper.new(@suite)
+        with_dtr_master do
+          # inject testcase as late as possible, for in ruby world there is lots hacks added to TestCase#run method,
+          # DTR should be the last one to add dtr injection chain into run method
+          Test::Unit::TestCase.send(:include, TestUnit::TestCaseInjection)
+          begin
+            run_suite_without_dtr_injection
+          ensure
+            Test::Unit::TestCase.reject_dtr
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dtr/test_unit/thread_safe_test_result.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'drb'
+
+module DTR
+  module TestUnit
+    class ThreadSafeTestResult
+      include DRbUndumped
+
+      def initialize(rs)
+        @mutex = Mutex.new
+        @rs = rs
+      end
+
+      def to_s
+        @mutex.synchronize do
+          @rs.to_s
+        end
+      end
+
+      def method_missing(method, *args, &block)
+        @mutex.synchronize do
+          @rs.send(method, *args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/dtr/test_unit/worker_club.rb

@@ -0,0 +1,72 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'singleton'
+
+module DTR
+  module TestUnit
+    class WorkerClub
+      include Singleton
+
+      DEFAULT_RUN_TEST_TIMEOUT = 60 #seconds
+
+      def start_thread(drb_runner, remote_runner)
+        thread = Thread.new(drb_runner, remote_runner) do |local, remote|
+          local.run_test_on(remote, timeout)
+        end
+        thread[:started_on] = Time.now
+        workers.add(thread)
+      end
+
+      # Performs a wait on all the currently running threads and kills any that take
+      # too long. It waits by ENV['RUN_TEST_TIMEOUT'] || 60 seconds
+      def graceful_shutdown
+        while reap_dead_workers("shutdown") > 0
+          DTR.info "Waiting for #{workers.list.length} threads to finish, could take #{timeout} seconds."
+          sleep timeout / 60
+        end
+      end
+
+      def timeout
+        ENV['RUN_TEST_TIMEOUT'] || DEFAULT_RUN_TEST_TIMEOUT
+      end
+
+      private
+      def workers
+        @workers ||= ThreadGroup.new
+      end
+
+      # Used internally to kill off any worker threads that have taken too long
+      # to complete processing. It returns the count of workers still active
+      # after the reap is done. It only runs if there are workers to reap.
+      def reap_dead_workers(reason='unknown')
+        if workers.list.length > 0
+          DTR.info "Reaping #{workers.list.length} threads because of '#{reason}'"
+          error_msg = "#{Time.now}: WorkerClub timed out this thread: #{reason}"
+          mark = Time.now
+          workers.list.each do |worker|
+            worker[:started_on] = Time.now if not worker[:started_on]
+
+            if mark - worker[:started_on] > timeout
+              DTR.info "Thread #{worker.inspect} is too old, killing."
+              worker.raise(TimeoutError.new(error_msg))
+            end
+          end
+        end
+
+        return workers.list.length
+      end
+    end
+  end
+end