dtr 0.0.4 → 1.0.0

data/lib/dtr/agent/runner.rb

@@ -0,0 +1,87 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    class Runner
+      include DRbUndumped
+      include Service::Runner
+
+      def self.start(name, env)
+        self.new(name, env).start
+        DRb.thread.join if DRb.thread
+      end
+
+      attr_reader :name, :identifier
+
+      def initialize(name, env)
+        @name = name
+        @identifier = env[:identifier]
+        @env = env
+      end
+      
+      def start
+        #start service first, so that all logs can be sync with master process
+        start_service
+        DTR.info("=> Starting runner #{name} at #{Dir.pwd}, pid: #{Process.pid}")
+        init_environment
+        provide
+        DTR.info {"=> Runner #{name} provided"}
+      rescue Exception
+        DTR.error($!.message)
+        DTR.error($!.backtrace.join("\n"))
+      end
+
+      def init_environment
+        DTR.info {"#{name}: Initialize working environment..."}
+        ENV['DTR_RUNNER_NAME'] = name
+
+        @env[:libs].select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
+          $LOAD_PATH << lib
+          DTR.debug {"#{name}: appended lib: #{lib}"}
+        end
+        DTR.info {"#{name}: libs loaded"}
+        DTR.debug {"#{name}: $LOAD_PATH: #{$LOAD_PATH.inspect}"}
+
+        @env[:files].each do |f|
+          begin
+            load f unless f =~ /^-/
+            DTR.debug {"#{name}: loaded #{f}"}
+          rescue LoadError => e
+            DTR.error {"#{name}: No such file to load -- #{f}"}
+            DTR.debug {"Environment: #{@env}"}
+          end
+        end
+        DTR.info {"#{name}: test files loaded"}
+      end
+
+      def run(test, result, &progress_block)
+        DTR.debug {"#{name}: running #{test}..."}
+        Agent::TestCase.new(test, result, &progress_block).run
+        DTR.debug {"#{name}: done #{test}"}
+      ensure
+        provide
+        DTR.debug {"=> Runner #{name} provided"}
+      end
+
+      def provide
+        provide_runner(self)
+      end
+
+      def to_s
+        "Runner #{@name}"
+      end
+    end
+  end
+end

data/lib/dtr/agent/sync_codebase.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncCodebase
+    module WorkingEnvExt
+      include SyncService
+
+      def self.included(base)
+        base.alias_method_chain :setup_env, :sync_codebase
+        base.alias_method_chain :working_dir, :sync_codebase
+      end
+
+      def setup_env_with_sync_codebase(setup_env_cmd)
+        unless same_working_dir_with_master_process?
+          Dir.chdir(working_dir_without_sync_codebase) do
+            sync_codebase
+          end
+        end
+        setup_env_without_sync_codebase(setup_env_cmd)
+      end
+
+      def working_dir_with_sync_codebase
+        same_working_dir_with_master_process? ? Dir.pwd : File.join(working_dir_without_sync_codebase, package_name)
+      end
+
+      private
+      def same_working_dir_with_master_process?
+        self[:host] == Socket.gethostname && self[:pwd] == Dir.pwd
+      end
+    end
+  end
+end

data/lib/dtr/agent/sync_logger.rb

@@ -0,0 +1,70 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncLogger
+
+    # Synchronizer loads SyncLogger provided by master process from Rinda server.
+    # Any process need sync logs with master process should start service first,
+    # so that local logger could be replaced by SyncLogger.
+    # For logs would be sent back to master process, all messages would be sent
+    # as string message. The message logged as block would be converted to string
+    # message.
+    module Synchronizer
+      def self.included(base)
+        base.alias_method_chain :start_service, :sync_logger
+      end
+
+      def start_service_with_sync_logger
+        start_service_without_sync_logger
+        if logger_tuple = lookup_ring.read_all([:logger, nil]).first
+          sync_logger = logger_tuple[1]
+          DTR.logger = MessageDecoratedLogger.new(sync_logger)
+        end
+      end
+    end
+
+    class MessageDecoratedLogger
+      include MessageDecorator
+
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def debug(message=nil, &block)
+        with_decorating_message(:debug, message, &block)
+      end
+
+      def info(message=nil, &block)
+        with_decorating_message(:info, message, &block)
+      end
+
+      def error(message=nil, &block)
+        with_decorating_message(:error, message, &block)
+      end
+
+      def level
+        @logger_level ||= @logger.level
+      end
+
+      private
+      def with_decorating_message(level, msg, &block)
+        raise 'Should not use block to send log remotely' if block_given?
+        @logger.send(level, decorate_message(msg))
+      end
+    end
+  end
+
+  Service::Rinda.send(:include, SyncLogger::Synchronizer)
+end

data/lib/dtr/agent/test_case.rb

@@ -0,0 +1,53 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    class UnknownTestError < StandardError
+    end
+    class TestCase
+      def initialize(test, result, &progress_block)
+        @test = test
+        @result = result
+        @progress_block = progress_block
+      end
+
+      def run
+        if @test.is_a?(DRb::DRbUnknown)
+          add_error(UnknownTestError.new("No such test loaded: #{@test.name}"))
+        else
+          @test.run(@result, &@progress_block)
+        end
+      rescue DRb::DRbConnError => e
+        msg = "Rescued DRb::DRbConnError(#{e.message}), while running test: #{test}. The master process may be stopped."
+        DTR.do_println(msg)
+        DTR.info {msg}
+      rescue Exception => e
+        unexpected_error(e)
+      end
+
+      def unexpected_error(e)
+        DTR.error "Unexpected exception: #{e.message}"
+        DTR.error e.backtrace.join("\n")
+        add_error(e)
+      end
+
+      def add_error(e)
+        @result.add_error(Test::Unit::Error.new(@test.name, RemoteError.new(e)))
+        @result.add_run
+        @progress_block.call(Test::Unit::TestCase::FINISHED, @test.name)
+      end
+    end
+  end
+end

data/lib/dtr/agent/test_unit.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test/unit'
+require 'test/unit/testcase'
+
+module DTR
+  module Agent
+    module TestCaseExt
+      include MessageDecorator
+      def self.included(base)
+        base.alias_method_chain :add_failure, :decorating_source
+        base.alias_method_chain :add_error, :decorating_source
+      end
+
+      def add_error_with_decorating_source(exception)
+        add_error_without_decorating_source(DTR::RemoteError.new(exception))
+      end
+
+      def add_failure_with_decorating_source(message, all_locations=caller())
+        add_failure_without_decorating_source(decorate_message(message, 'Assertion failure'), all_locations)
+      end
+    end
+  end
+end
+
+# set run to true first, so that test auto runner wouldn't work
+Test::Unit.run = true
+Test::Unit::TestCase.send(:include, DTR::Agent::TestCaseExt)

data/lib/dtr/agent/worker.rb

@@ -0,0 +1,89 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+
+  module Agent
+    # Worker works during one dtr test task running.
+    # Worker manages Herald & Runner processes life cycle.
+    class Worker
+      def initialize(runner_names, agent_env_setup_cmd)
+        @runner_names = runner_names.is_a?(Array) ? runner_names : [runner_names.to_s]
+        @agent_env_setup_cmd = agent_env_setup_cmd
+        @runner_pids = []
+        @herald = nil
+        @working_env_key = :working_env
+        @env_store = EnvStore.new
+      end
+
+      def launch
+        DTR.info {"=> Agent worker started at: #{Dir.pwd}, pid: #{Process.pid}"}
+        setup
+        begin
+          run
+        ensure
+          teardown
+          DTR.info {"Agent worker is dieing"}
+        end
+      end
+
+      private
+      def setup
+        @env_store[@working_env_key] = nil
+      end
+
+      def teardown
+        unless @runner_pids.blank?
+          @runner_pids.each{ |pid| DTR.kill_process pid }
+          DTR.info {"=> All runners(#{@runner_pids.join(", ")}) were killed." }
+          @runner_pids = []
+        end
+        if @herald
+          DTR.kill_process @herald
+          @herald = nil
+          DTR.info {"=> Herald is killed."}
+        end
+      end
+
+      def run
+        herald
+        runners
+        DTR.info {"=> All agent worker sub processes exited."}
+      end
+
+      def herald
+        @herald = DTR.fork_process { Herald.new @working_env_key, @agent_env_setup_cmd, @runner_names }
+        Process.waitpid @herald
+        exit(-1) unless $?.exitstatus == 0
+      end
+
+      def runners
+        working_env = @env_store[@working_env_key]
+
+        @runner_names.each do |name|
+          @runner_pids << DTR.fork_process {
+            at_exit {
+              # exit anyway, for DRb may hang on the process to be a deadwalk
+              exit!
+            }
+            working_env.within do
+              Runner.start name, working_env
+            end
+          }
+        end
+        Process.waitall
+      end
+    end
+  end
+end

data/lib/dtr/agent/working_env_ext.rb

@@ -0,0 +1,47 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+    # Provides working environment setup beheaviours for Herald & Runner process
+    module WorkingEnvExt
+      def within
+        ENV['DTR_MASTER_ENV'] = self[:dtr_master_env]
+        Dir.chdir(working_dir) do
+          yield
+        end
+      end
+
+      def setup_env(setup_env_cmd)
+        within do
+          Cmd.execute(setup_env_cmd || self[:agent_env_setup_cmd])
+        end
+      end
+
+      private
+      def escape(str)
+        str.gsub(/[^a-zA-Z0-9]/, '_')
+      end
+
+      def working_dir
+        return @working_dir if defined?(@working_dir)
+        project_specific_len = 20
+        project_name = self[:pwd].length > project_specific_len ? self[:pwd][-project_specific_len..-1] : self[:pwd]
+        @working_dir = File.join(escape(self[:host]), escape(project_name))
+        FileUtils.mkdir_p(@working_dir)
+        @working_dir
+      end
+    end
+  end
+end

data/lib/dtr/facade.rb

@@ -0,0 +1,65 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Facade
+    def start_agent
+      launch_agent(DTR_AGENT_OPTIONS[:runners], DTR_AGENT_OPTIONS[:agent_env_setup_cmd])
+    end
+
+    def launch_agent(names, setup=nil)
+      require 'dtr/agent'
+      names = names || "DTR(#{Time.now})"
+      DTR::Agent.start(names, setup)
+    end
+
+    def lib_path
+      File.expand_path(File.dirname(__FILE__) + '/../')
+    end
+
+    def broadcast_list=(list)
+      require 'dtr/shared'
+      DTR.configuration.broadcast_list = list
+      DTR.configuration.save
+    end
+
+    def agent_listen_port=(port)
+      require 'dtr/shared'
+      DTR.configuration.agent_listen_port = port
+      DTR.configuration.save
+    end
+
+    def group=(group)
+      require 'dtr/shared'
+      DTR.configuration.group = group
+      DTR.configuration.save
+    end
+
+    def monitor
+      require 'dtr/monitor'
+      DTR.logger('dtr_monitor.log')
+      Monitor.new.start
+    end
+
+    # For safe fork & kill sub process, should use Process.kill and Process.fork
+    # At least have problem on ruby 1.8.6 114 with Kernel#kill & fork
+    def kill_process(pid)
+      Process.kill 'TERM', pid rescue nil
+    end
+
+    def fork_process(&block)
+      Process.fork(&block)
+    end
+  end
+end