xli-dtr 0.0.4

data/lib/dtr/base.rb

@@ -0,0 +1,172 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pstore'
+
+require 'logger'
+
+class Array
+  def blank?
+    empty?
+  end
+end
+
+class NilClass
+  def blank?
+    true
+  end
+end
+
+module DTR
+  
+  MESSAGE_KEY = :message
+  
+  def logger
+    return DTROPTIONS[:logger] if DTROPTIONS[:logger]
+    DTROPTIONS[:logger] = if DTROPTIONS[:log_level] == Logger::DEBUG
+      Logger.new(STDOUT)
+    else
+      log_file = if File.exist?('log')
+        File.join("log", DTROPTIONS[:log_file] || 'dtr.log')
+      else
+        DTROPTIONS[:log_file] || 'dtr.log'
+      end
+      Logger.new("./#{log_file}", 1, 5*1024*1024)
+    end
+    DTROPTIONS[:logger].datetime_format = "%m-%d %H:%M:%S"
+    DTROPTIONS[:logger].level = DTROPTIONS[:log_level] || Logger::INFO
+    DTROPTIONS[:logger]
+  end
+
+  def debug(message=nil, &block)
+    output(:debug, message, &block)
+  end
+  
+  def info(message=nil, &block)
+    output(:info, message, &block)
+  end
+  
+  def error(message=nil, &block)
+    output(:error, message, &block)
+  end
+  
+  def output(level, msg=nil, &block)
+    logger.send(level) do
+      message = block_given? ? block.call : msg.to_s
+      EnvStore.new << [MESSAGE_KEY, "[#{Process.pid}-#{level.to_s.upcase}] #{message}"] if DTROPTIONS[:run_with_monitor]
+      message
+    end
+  end
+  
+  def silent?
+    logger.level == Logger::ERROR
+  end
+  
+  def with_monitor
+    DTROPTIONS[:run_with_monitor] = true
+    EnvStore.new[MESSAGE_KEY] = []
+    yield
+  rescue Exception => e
+    info {"stopping by Exception => #{e.class.name}, message => #{e.message}"}
+    wait_times = 0
+    until EnvStore.new[MESSAGE_KEY].empty? || wait_times > 14
+      wait_times += 1
+      sleep(1)
+    end
+    raise e
+  end
+  
+  module_function :debug, :info, :error, :output, :with_monitor, :logger, :silent?
+  
+  class CmdInterrupt < StandardError; end
+
+  class Cmd
+    def self.execute(cmd)
+      return true if cmd.nil? || cmd.empty?
+      DTR.info {"Executing: #{cmd.inspect}"}
+      output = %x[#{cmd} 2>&1]
+      DTR.info {"Execution is done, status: #{$?.exitstatus}"}
+      DTR.error {"#{cmd.inspect} output:\n#{output}"} if $?.exitstatus != 0
+      $?.exitstatus == 0
+    end
+  end
+  
+  class EnvStore
+
+    FILE_NAME = '.dtr_env_pstore'
+
+    def self.destroy
+      File.delete(FILE_NAME) if File.exist?(FILE_NAME)
+    end
+
+    def [](key)
+      return nil unless File.exist?(FILE_NAME)
+      
+      repository = PStore.new(FILE_NAME)
+      repository.transaction(true) do
+        repository[key]
+      end
+    end
+
+    def []=(key, value)
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        repository[key] = value
+      end
+    end
+    
+    def <<(key_value)
+      key, value = key_value
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        repository[key] = (repository[key] || []) << value
+      end
+    end
+    
+    def shift(key)
+      repository = PStore.new(FILE_NAME)
+      repository.transaction do
+        if array = repository[key]
+          array.shift
+          repository[key] = array
+        end
+      end
+    end
+  end
+
+  class WorkingEnv
+  
+    @@current = nil
+    def self.refresh
+      @@current = self.new
+    end
+    
+    def self.current
+      @@current
+    end
+    
+    def initialize
+      files = (defined?($argv_dup) ? $argv_dup : []).dup
+      @env = {:libs => $LOAD_PATH.dup, :files => files, :created_at => Time.now.to_s, :dtr_master_env => ENV['DTR_MASTER_ENV'], :identifier => "#{Time.now.to_s}:#{rand}:#{object_id}"}
+    end
+    
+    def [](key)
+      @env[key]
+    end
+    
+    def to_s
+      @env.inspect
+    end
+  end
+end

data/lib/dtr/raketasks.rb

@@ -0,0 +1,69 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "rubygems"
+require 'dtr'
+require 'rake/testtask'
+
+module DTR
+  class MPTask < Rake::TestTask
+    attr_accessor :processes, :runner_options, :start_server
+    
+    def define
+      @libs.unshift DTR.lib_path
+      lib_path = @libs.join(File::PATH_SEPARATOR)
+
+      desc "Run tests" + (@name==:test ? "" : " for #{@name}")
+      task @name do
+        DTR.start_server_daemon_mode if start_server?
+        start_runners
+        run_code = ''
+        begin
+          RakeFileUtils.verbose(@verbose) do
+            run_code = rake_loader
+            @ruby_opts.unshift( "-I#{lib_path}" )
+            @ruby_opts.unshift( "-w" ) if @warning
+            
+            ruby @ruby_opts.join(" ") +
+              " \"#{run_code}\" " +
+              file_list.unshift('dtr/test_unit_injection.rb').collect { |fn| "\"#{fn}\"" }.join(' ') +
+              " #{option_list}"
+          end
+        ensure
+          DTR.stop_runners_daemon_mode rescue nil
+          if start_server?
+            DTR.stop_server_daemon_mode rescue nil
+          end
+        end
+      end
+      self
+    end
+    
+    def processes
+      @processes ? @processes.to_i : 2
+    end
+    
+    def start_server?
+      defined?(@start_server) ? @start_server : true
+    end
+    
+    private
+    def start_runners
+      return if self.processes.to_i <= 0
+      runner_names = []
+      self.processes.to_i.times {|i| runner_names << "runner#{i}"}
+      %x[dtr -r #{runner_names.join(',')} -D #{runner_options}]
+    end
+  end
+end

data/lib/dtr/runner.rb

@@ -0,0 +1,270 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/base'
+require 'dtr/service_provider'
+require 'test/unit'
+require 'drb'
+
+class Test::Unit::TestCase
+  alias_method :add_error_without_hack, :add_error
+  def add_error(exception)
+    add_error_without_hack(DTR::RunnerRuntimeException.new(exception))
+  end
+
+  alias_method :add_failure_without_hack, :add_failure
+  def add_failure(message, all_locations=caller())
+    add_failure_without_hack(DTR.decorate_error_message(message, 'Assertion failure'), all_locations)
+  end
+end
+
+module DTR
+  
+  def service_provider
+    ServiceProvider.new
+  end
+  
+  module_function :service_provider
+
+  class RunnerAgent
+    
+    def self.start(runner_names=["Distributed Test Runner"], setup_cmd=nil)
+      DTR.with_monitor do 
+        new(runner_names, setup_cmd).launch
+      end
+    end
+    
+    def initialize(runner_names, setup_cmd)
+      @runner_names = runner_names.is_a?(Array) ? runner_names : [runner_names.to_s]
+      @setup_cmd = setup_cmd || ""
+      @runner_pids = []
+      @herald = nil
+      @working_env_key = :working_env
+      @env_store = EnvStore.new
+      @agent_pid = Process.pid
+      at_exit {
+        if Process.pid == @agent_pid
+          DTR.info "*** Runner agent is stopping ***"
+          kill_all_runners
+          if @herald
+            Process.kill 'KILL', @herald rescue nil
+            DTR.info "=> Herald is killed." 
+          end
+          if @heart
+            Process.kill 'KILL', @heart rescue nil
+            DTR.info "=> Heartbeat is stopped." 
+          end
+          DTR.info "*** Runner agent stopped ***"
+        end
+      }
+    end
+    
+    def launch
+      DTR.info "=> Runner agent started at: #{Dir.pwd}, pid: #{Process.pid}"
+      @heart = drb_fork { Heart.new }
+      @herald = drb_fork { Herald.new @working_env_key }
+      working_env = {}
+      @env_store[@working_env_key] = nil
+      loop do
+        if @env_store[@working_env_key] && working_env[:identifier] != @env_store[@working_env_key][:identifier]
+          working_env = @env_store[@working_env_key]
+
+          DTR.info "=> Got new working environment created at #{working_env[:created_at]}"
+
+          kill_all_runners
+          ENV['DTR_MASTER_ENV'] = working_env[:dtr_master_env]
+
+          if Cmd.execute(@setup_cmd)
+            @runner_names.each do |name| 
+              @runner_pids << drb_fork { Runner.start name, working_env }
+            end
+          else
+            DTR.info {'No runners started.'}
+          end
+        end
+        sleep(2)
+      end
+    end
+    
+    private
+    
+    def kill_all_runners
+      unless @runner_pids.blank?
+        @runner_pids.each{ |pid| Process.kill 'KILL', pid rescue nil }
+        DTR.info "=> All runners(#{@runner_pids.join(", ")}) were killed." 
+        @runner_pids = []
+      end
+    end
+    
+    def drb_fork
+      Process.fork do
+        at_exit {
+          DRb.stop_service
+          exit!
+        }
+        begin
+          yield
+        rescue Interrupt => e
+          raise e
+        rescue SystemExit => e
+          raise e
+        rescue Exception => e
+          DTR.error "Got an Exception #{e.message}:"
+          DTR.error e.backtrace.join("\n")
+          raise e
+        end
+      end
+    end
+  end
+  
+  class Heart
+    def initialize(key=MESSAGE_KEY)
+      @key = key
+      @env_store = EnvStore.new
+      @provider = DTR.service_provider
+      beat
+    end
+    
+    def beat
+      loop do
+        begin
+          if @env_store[@key].blank?
+            @provider.send_message('---/V---')
+          else
+            while message = @env_store[@key].first
+              @provider.send_message(message)
+              @env_store.shift(@key)
+            end
+          end
+          sleep_any_way
+        rescue => e
+          DTR.info "Heart lost DTR Server(#{e.message}), going to sleep 10 sec..."
+          @env_store[@key] = []
+          sleep_any_way
+        end
+      end
+    end
+    
+    private
+    def sleep_any_way
+      sleep(10)
+    rescue Exception
+    end
+  end
+  
+  class Herald
+    
+    def initialize(key)
+      @key = key
+      @env_store = EnvStore.new
+      @env_store[@key] = nil
+      @provider = DTR.service_provider
+      start_off
+    end
+    
+    def start_off
+      loop do
+        DTR.info "=> Herald starts off..."
+        begin
+          working_env = @provider.working_env
+          DTR.debug { "working env: #{working_env.inspect}" }
+          if working_env[:files].blank?
+            DTR.error "No test files need to load?(working env: #{working_env.inspect})"
+          else
+            @env_store[@key] = working_env if @env_store[@key].nil? || @env_store[@key][:identifier] != working_env[:identifier]
+            @provider.wait_until_teardown
+          end
+
+          sleep(2)
+        rescue => e
+          DTR.info "Herald lost DTR Server(#{e.message}), going to sleep 5 sec..."
+          sleep(5)
+        end
+      end
+    end
+  end
+  
+  class Runner
+    include DRbUndumped
+    
+    def self.start(name, env)
+      DTR.info "#{name}: Initialize working environment..."
+      env[:libs].select{ |lib| !$LOAD_PATH.include?(lib) && File.exists?(lib) }.each do |lib|
+        $LOAD_PATH << lib
+        DTR.debug {"#{name}: appended lib: #{lib}"}
+      end
+      DTR.info "#{name}: libs loaded"
+      DTR.debug {"#{name}: $LOAD_PATH: #{$LOAD_PATH.inspect}"}
+      
+      env[:files].each do |f|
+        begin
+          load f unless f =~ /^-/
+          DTR.debug {"#{name}: loaded #{f}"}
+        rescue LoadError => e
+          DTR.error "#{name}: No such file to load -- #{f} (Environment: #{env.inspect})"
+        end
+      end
+      DTR.info "#{name}: test files loaded"
+
+      @provider = DTR.service_provider
+
+      @provider.provide(self.new(@provider, name, env[:identifier]))
+      DTR.info "=> Runner #{name} provided"
+      DRb.thread.join if DRb.thread
+    end
+    
+    attr_reader :name, :identifier
+    
+    def initialize(provider, name, identifier)
+      Test::Unit.run = true
+      @name = name
+      @provider = provider
+      @identifier = identifier
+      @started = []
+      @run_finished = []
+    end
+    
+    def run(test, result, &progress_block)
+      DTR.debug {"#{name}: running #{test}..."}
+      @started << test.name
+      test.run(result, &progress_block)
+    rescue DRb::DRbConnError => e
+      DTR.info{ "Rescued DRb::DRbConnError(#{e.message}), while running test: #{test.name}. The master process may be stopped." }
+    rescue Exception => e
+      DTR.error {"Unexpected exception: #{e.message}"}
+      DTR.error {e.backtrace.join("\n")}
+      result.add_error(Test::Unit::Error.new(test.name, e))
+      result.add_run
+      progress_block.call(Test::Unit::TestCase::FINISHED, test.name)
+    ensure
+      DTR.debug {"#{name}: done #{test}"}
+      @run_finished << test.name
+      @provider.provide(self)
+    end
+    
+    def reboot
+      DTR.info "#{self} is rebooting. Ran #{@started.size} tests, finished #{@run_finished.size}."
+      @provider.provide(self)
+    end
+    
+    def shutdown
+      DTR.info "#{self} is shutting down. Ran #{@started.size} tests, finished #{@run_finished.size}."
+      @provider.stop_service rescue exit!
+    end
+    
+    def to_s
+      "Runner #{@name}"
+    end
+  end
+end

data/lib/dtr/service_provider.rb

@@ -0,0 +1,160 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/base'
+require 'drb'
+require 'rinda/ring'
+require 'rinda/tuplespace'
+require 'socket'
+
+module DTR
+  def decorate_error_message(msg, source=nil)
+    source ? "#{source} from #{Socket.gethostname}: #{msg}" : "From #{Socket.gethostname}: #{msg}"
+  end
+  
+  module_function :decorate_error_message
+
+  class RunnerRuntimeException < StandardError
+    def initialize(e)
+      super(DTR.decorate_error_message(e.message, e.class.name))
+      set_backtrace(e.backtrace)
+    end
+  end
+
+  class ServiceProvider
+  
+    def self.broadcast_list=(list)
+      EnvStore.new[:broadcast_list] = list
+    end
+    
+    def self.port=(port)
+      EnvStore.new[:port] = port
+    end
+    
+    PORT = 3344
+    BROADCAST_LIST = []
+    
+    def initialize
+      DTR.info "-- Initializing drb service..."
+      env_store = EnvStore.new
+      (env_store[:broadcast_list] || ['localhost']).each do |broadcast|
+        BROADCAST_LIST << broadcast.untaint
+        DTR.info "-- Added broadcast: #{broadcast}"
+      end
+      DTR.info "-- Server port: #{server_port}"
+      DRb.start_service
+    end
+    
+    # start DTR server
+    def start
+      env_store = EnvStore.new
+      DTR.info '-- Booting DTR server...'
+      Rinda::RingServer.new Rinda::TupleSpace.new, server_port
+      DTR.info "-- DTR server started on port #{server_port}"
+      #set safe level to 1 here, now, runner can't set to 1, cause test should can do anything
+      #......
+      $SAFE = 1 unless $DEBUG   # disable eval() and friends
+      # Wait until the user explicitly kills the server.
+      DRb.thread.join
+    end
+    
+    def provide(runner)
+      renewer = Rinda::SimpleRenewer.new
+      tuple = [:name, 'DTR::Runner'.to_sym, runner.freeze, "DTR remote runner #{Process.pid}-#{runner.name}"]
+      lookup_ring.write(tuple, renewer)
+    end
+    
+    def send_message(message)
+      lookup_ring.write [:agent_heartbeat, Socket.gethostname, message, Time.now], 2
+    end
+    
+    def lookup_runner
+      lookup_ring.take([:name, 'DTR::Runner'.to_sym, nil, nil])[2]
+    end
+    
+    def runners
+      lookup_ring.read_all([:name, 'DTR::Runner'.to_sym, nil, nil]).collect {|rt| rt[2]}
+    end
+    
+    def monitor
+      working_env_monitor = lookup_ring.notify(nil, [:working_env, nil])
+      Thread.start do
+        DTR.info("Current work environment: #{working_env.inspect}")
+        working_env_monitor.each { |t| DTR.info t.inspect }
+      end
+      if DTROPTIONS[:log_level] == Logger::DEBUG
+        runner_monitor = lookup_ring.notify(nil, [:name, 'DTR::Runner'.to_sym, nil, nil])
+        Thread.start do
+          runner_monitor.each { |t| DTR.debug t.inspect }
+        end
+      end
+      agent_heartbeat_monitor = lookup_ring.notify("write", [:agent_heartbeat, nil, nil, nil])
+      Thread.start do
+        colors = {}
+        base = 30
+        agent_heartbeat_monitor.each do |t|
+          host, message, time = t[1][1..3]
+          colors[host] = base+=1 unless colors[host]
+          message = "\e[1;31m#{message}\e[0m" if message =~ /-ERROR\]/
+          DTR.info "#{time.strftime("[%I:%M:%S%p]")} \e[1;#{colors[host]};1m#{host}\e[0m: #{message}"
+        end
+      end
+      DRb.thread.join
+    end
+    
+    def wait_until_teardown
+      lookup_ring.notify(nil, [:working_env, nil]).pop
+    end
+
+    def working_env
+      lookup_ring.read([:working_env, nil])[1]
+    end
+    
+    def setup_working_env(env)
+      clear_workspace
+      lookup_ring.write [:working_env, env]
+    end
+    
+    def teardown_working_env
+      clear_workspace
+    end
+
+    def start_service
+      DRb.start_service
+    end
+    
+    def stop_service
+      DRb.stop_service
+    end
+    
+    def clear_workspace
+      lookup_ring.read_all([:working_env, nil]).size.times do
+        lookup_ring.take [:working_env, nil] rescue nil
+      end rescue nil
+      runners.size.times do
+        lookup_runner.shutdown rescue nil
+      end
+    end
+
+    private
+    def server_port
+      env_store = EnvStore.new
+      env_store[:port].to_i > 0 ? env_store[:port].to_i : PORT
+    end
+
+    def lookup_ring
+      Rinda::TupleSpaceProxy.new(Rinda::RingFinger.new(BROADCAST_LIST, server_port).lookup_ring_any)
+    end
+  end
+end