dtr 0.0.4 → 1.0.0

data/TODO

@@ -1,8 +1,14 @@
 = DTR Project -- To Do List
 
-Send suggestions for this list to swing1979@gmail.com
+Send suggestions for this list to iam@li-xiao.com
 
 === To Do
-  39. class doc(ri)?
+
+=== TBD for next release
+  should trigger started notification on test runner in agent Herald, for some tools are using this hook(e.g. inbrowser_test) to setup test environment.
+  agent should wakeup when worker is dead, so that it could be picked up by another master process
+  auto prepare database for each runner, so that run tests in multi-processes could be easy setup.
+  provide a way to monitor agent runner efficiency
+  use system instead of fork to launch sub process in agent, which makes dtr support windows and jruby
   
 (moved DONE list to CHANGES file)

data/bin/dtr

@@ -11,10 +11,13 @@ end
 NOTES = <<-NOTES
 ----------------
 Notes: 
-  * The default value of server address is 'localhost'. If DTR Server start with 'localhost', it would not be found by remote machine. If you run the runner agent on remote machine, you should start the server and runner agent with same server address specified, e.g. dtr -s -a 10.18.1.1, dtr -r r1,r2 -a 10.18.1.1
-  * Runners specified by -r option will be started in different processes by the runner agent.
-  * You need install gem daemons to use -D option.
-  * When specify -D option, dtr_runners.pid/dtr_server.pid will be created in the directory to store the daemon process pid. And the output of the process will be print into *.output file.
+  * DTR would always add 'localhost' into broadcast list.
+  * Agent start by specifying runners by -r option, every runner would be started in different process for running test. e.g. dtr -r runner1,runner2
+  * DTR master environment options:
+    * DTR_MASTER_ENV: this variable would be copied into agent process for sharing info between master and agents. Normally used in agent setup environment command.
+    * DTR_AGENT_ENV_SETUP_CMD: this variable would be applied as agent setup environment command when agent have no setup environment command specified by --setup option.
+    * DTR_LOG_LEVEL: master process logger level, e.g. ENV['DTR_LOG_LEVEL'] = Logger::DEBUG. Agent process logs would be output in master process log file, so setting this option also changes agent logger level.
+    * DTR_RUNNER_NAME: this environment variable would be provided in runner process for test environment to get the runner name to setup, e.g. setting up database configuration.
 
 DTR is a distributed test runner program for decreasing time of running ruby tests based on ruby 'test/unit' package.
 For additional information, see http://dtr.rubyforge.org/
@@ -24,60 +27,39 @@ opts = OptionParser.new do |opts|
   opts.banner = "DTR usage: #{$0} [options]"
   opts.separator ""
   opts.separator "Synopsis:"
-  opts.separator "dtr -s"
-  opts.separator "dtr -r runner1_name,runner2_name"
-  opts.separator "dtr -a dtr_server_ip -m"
+  opts.separator "dtr -g group_name -r runner1_name,runner2_name"
+  opts.separator "dtr -a broadcast_ip -m"
   opts.separator ""
   opts.separator "Options:"
   
-  opts.on_tail("-m", "--monitor", "Monitor the status of the dtr rinda server, e.g. dtr -a 10.11.1.2 -m") do
-    DTROPTIONS[:monitor] = true
-  end
-  
-  opts.on_tail("-c", "--clean_server", "Clean server environment, shutdown all runners registered.") do
-    DTR.clean_server
-  end
-  
-  opts.on_tail("-b", "--be_silent", "Only show error messages") do
-    DTROPTIONS[:log_level] = Logger::ERROR
+  opts.on_tail("-m", "--monitor", "Monitor the status of the dtr agents and master processes. Used for testing your dtr grid environment. CAUTION! monitoring agents causes all idle agents hang on by the monitor process.") do
+    $monitor = true
   end
 
-  opts.on_tail("-s", "--server", "Start DTR service server. There must be a DTR server running.") do
-    DTROPTIONS[:start_server] = true
+  opts.on("-p", "--port PORT",  "Port number of DTR agent listening. Default is 7788.") do |port|
+    DTR.agent_listen_port = port
   end
   
-  opts.on("-p", "--port PORT",  "Port number DTR will listen to. Default is 3344.") do |port|
-    	DTR.port = port
+  opts.on("-g", "--group GROUP_NAME",  "If you have several DTR grids working for different project or environment, you should group your agents in different names for different usages. Default is none.") do |group_name|
+    DTR.group = group_name
   end
   
   opts.on("-r runner1_name,runner2_name", Array, "Start DTR test runner agent with unique runner names.") do |names|
-    DTROPTIONS[:names] = names.collect{|name| name.untaint}
+    DTR_AGENT_OPTIONS[:runners] = names.collect{|name| name.untaint}
   end
   
-  opts.on("-a", "--server_address ADDRESS", "Specify dtr server address, domain name or ip address, e.g. 192.168.0.1. Default is 'localhost'.") do |address|
+  opts.on("-a", "--broadcast_address ADDRESS", "Specify broadcast address for looking up dtr agent service, e.g. 192.168.255.255. Default is 'localhost'. DTR master and monitor would need this.") do |address|
     if (!address.nil?) && (!address.empty?)
       DTR.broadcast_list = [address]
     end
   end
 
-  opts.on("-i", "--setup COMMAND", "Set command for initializing test runner test environment, e.g. 'rake db:test:prepare'. Default is do nothing.") do |command|
-    DTROPTIONS[:setup] = command.untaint
-  end
-  
-  opts.on_tail("-D", "--daemon", "Start server/runners in daemon mode. Gem 'daemons' must be installed") do
-    DTROPTIONS[:daemon] = true
-  end
-  
-  opts.on_tail("-S", "--stop_server", "Stop server run in daemon mode.") do
-    DTR.stop_server_daemon_mode
+  opts.on("-i", "--setup COMMAND", "Set command for initializing test runner test environment, e.g. 'rake db:test:prepare'. Default is do nothing. You also can specify DTR_AGENT_ENV_SETUP_CMD in your master process environment to let all agents setup same environment.") do |command|
+    DTR_AGENT_OPTIONS[:agent_env_setup_cmd] = command.untaint
   end
-  
-  opts.on_tail("-R", "--stop_runners", "Stop runners run in daemon mode.") do
-    DTR.stop_runners_daemon_mode
-  end
-  
-  opts.on_tail("-d", "--debug", "output debug log") do
-    DTROPTIONS[:log_level] = Logger::DEBUG
+
+  opts.on("-d", "--working_directory DIRECTORY", "Specify a directory as agent launching & working directory.") do |dir|
+    DTR_AGENT_OPTIONS[:launching_dir] = dir
   end
   
   opts.on_tail("-v", "--version", "Show version") do
@@ -99,26 +81,11 @@ if no_argv
   puts NOTES
 end
 
-if DTROPTIONS[:names]
-  if DTROPTIONS[:daemon]
-    DTR.start_runners_daemon_mode
-  else
-    DTR.start_runners
-  end
+if $monitor
+  DTR.monitor
 end
-
-if DTROPTIONS[:start_server]
-  if DTROPTIONS[:daemon]
-    DTR.start_server_daemon_mode
-  else
-    DTR.start_server
+if DTR_AGENT_OPTIONS[:runners] && !DTR_AGENT_OPTIONS[:runners].empty?
+  Dir.chdir(DTR_AGENT_OPTIONS[:launching_dir] || '.') do
+    DTR.start_agent
   end
 end
-
-if DTROPTIONS[:monitor]
-  if DTROPTIONS[:daemon]
-    puts "Can't start monitor with daemon mode."
-  else
-    DTR.monitor
-  end
-end

data/dtr.gemspec

@@ -1,17 +1,12 @@
 Gem::Specification.new do |spec|
   spec.name = 'dtr'
-  spec.version = "0.0.4"
+  spec.version = "1.0.0"
   spec.summary = "DTR is a distributed test runner to run tests on distributed computers for decreasing build time."
 
   #### Dependencies and requirements.
+  spec.files = ["lib/dtr/agent/brain.rb", "lib/dtr/agent/herald.rb", "lib/dtr/agent/runner.rb", "lib/dtr/agent/sync_codebase.rb", "lib/dtr/agent/sync_logger.rb", "lib/dtr/agent/test_case.rb", "lib/dtr/agent/test_unit.rb", "lib/dtr/agent/worker.rb", "lib/dtr/agent/working_env_ext.rb", "lib/dtr/agent.rb", "lib/dtr/facade.rb", "lib/dtr/master.rb", "lib/dtr/monitor.rb", "lib/dtr/raketasks.rb", "lib/dtr/shared/adapter.rb", "lib/dtr/shared/configuration.rb", "lib/dtr/shared/message_decorator.rb", "lib/dtr/shared/ruby_ext.rb", "lib/dtr/shared/service/agent.rb", "lib/dtr/shared/service/file.rb", "lib/dtr/shared/service/rinda.rb", "lib/dtr/shared/service/runner.rb", "lib/dtr/shared/service/working_env.rb", "lib/dtr/shared/service.rb", "lib/dtr/shared/sync_codebase/copiable_package.rb", "lib/dtr/shared/sync_codebase/master_ext.rb", "lib/dtr/shared/sync_codebase/package.rb", "lib/dtr/shared/sync_codebase/sync_service.rb", "lib/dtr/shared/sync_codebase.rb", "lib/dtr/shared/sync_logger.rb", "lib/dtr/shared/utils/cmd.rb", "lib/dtr/shared/utils/env_store.rb", "lib/dtr/shared/utils/logger.rb", "lib/dtr/shared/utils.rb", "lib/dtr/shared/working_env.rb", "lib/dtr/shared.rb", "lib/dtr/test_unit/drb_test_runner.rb", "lib/dtr/test_unit/injection.rb", "lib/dtr/test_unit/test_case_injection.rb", "lib/dtr/test_unit/test_suite_injection.rb", "lib/dtr/test_unit/testrunnermediator_injection.rb", "lib/dtr/test_unit/thread_safe_test_result.rb", "lib/dtr/test_unit/worker_club.rb", "lib/dtr/test_unit.rb", "lib/dtr/test_unit_injection.rb", "lib/dtr.rb", "bin/dtr", "CHANGES", "dtr.gemspec", "lib", "LICENSE.TXT", "Rakefile", "README.rdoc", "TODO"]
 
-  spec.add_dependency('daemons', '> 1.0.7')
-  #s.requirements << ""
-
-  #### Which files are to be included in this gem?  Everything!  (Except SVN directories.)
-
-  spec.files = FileList['lib/**/*.rb', 'lib/**/*.rake', 'bin/*', '[a-zA-Z]*'].to_a
-  p FileList['lib/**/*.rb', 'lib/**/*.rake', 'bin/*', '[a-zA-Z]*'].to_a
+  spec.test_files = ["test/acceptance/agent_working_env_test.rb", "test/acceptance/dtr_package_task_test.rb", "test/acceptance/general_test.rb", "test/acceptance/raketasks_test.rb", "test/acceptance/sync_codebase_test.rb", "test/acceptance/sync_logger_test.rb", "test/agent_helper.rb", "test/logger_stub.rb", "test/test_helper.rb", "test/unit/adapter_test.rb", "test/unit/configuration_test.rb", "test/unit/facade_test.rb", "test/unit/logger_test.rb", "test/unit/test_unit_test.rb", "test/unit/working_env_test.rb", "testdata/a_failed_test_case.rb", "testdata/a_file_system_test_case.rb", "testdata/a_test_case.rb", "testdata/a_test_case2.rb", "testdata/an_error_test_case.rb", "testdata/another_project", "testdata/another_project/passed_test_case.rb", "testdata/another_project/Rakefile", "testdata/hacked_run_method_test_case.rb", "testdata/is_required_by_a_test.rb", "testdata/lib", "testdata/lib/lib_test_case.rb", "testdata/package_task_test_rakefile", "testdata/Rakefile", "testdata/raketasks", "testdata/raketasks/Rakefile", "testdata/raketasks/success_test_case.rb", "testdata/scenario_test_case.rb", "testdata/setup_agent_env_test_case.rb", "testdata/sleep_3_secs_test_case.rb", "testdata/verify_dir_pwd", "testdata/verify_dir_pwd/Rakefile", "testdata/verify_dir_pwd/verify_dir_pwd_test_case.rb"]
 
   #### Load-time details: library and application (you will need one or both).
 
@@ -23,12 +18,14 @@ Gem::Specification.new do |spec|
 
   #### Documentation and testing.
 
-  spec.has_rdoc = false
+  spec.has_rdoc = true
+  spec.extra_rdoc_files = ["README.rdoc", "LICENSE.txt", "TODO", "CHANGES"]
+  spec.rdoc_options = ["--line-numbers", "--inline-source", "--main", "README.rdoc", "--title", "\"DTR -- Distributed Test Runner"]
 
   #### Author and project details.
 
   spec.author = "Li Xiao"
-  spec.email = "swing1979@gmail.com"
+  spec.email = "iam@li-xiao.com"
   spec.homepage = "http://github.com/xli/dtr/tree/master"
   spec.rubyforge_project = "dtr"
-end
+end

data/lib/dtr.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,100 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-DTRVERSION="0.0.4"
-DTROPTIONS = {} unless defined?(DTROPTIONS)
+require 'dtr/facade'
 
-require 'fileutils'
+DTRVERSION="1.0.0"
+DTR_AGENT_OPTIONS = {:runners => [], :agent_env_setup_cmd => nil}
 
 module DTR
-
-  def start_server
-    require 'dtr/service_provider'
-    ServiceProvider.new.start
-  end
-  
-  def start_runners
-    launch_runners(DTROPTIONS[:names], DTROPTIONS[:setup])
-  end
-  
-  def launch_runners(names, setup=nil)
-    require 'dtr/runner'
-    names = names || "DTR(#{Time.now})"
-    DTR::RunnerAgent.start(names, setup)
-  end
-  
-  def lib_path
-    File.expand_path(File.dirname(__FILE__))
-  end
-  
-  def broadcast_list=(list)
-    require 'dtr/service_provider'
-    ServiceProvider.broadcast_list = list
-  end
-  
-  def port=(port)
-    require 'dtr/service_provider'
-    ServiceProvider.port = port 
-  end
-  
-  def runners
-    require 'dtr/service_provider'
-    ServiceProvider.new.runners
-  end
-  
-  def monitor
-    require 'dtr/service_provider'
-    DTROPTIONS[:log_file] = 'dtr_monitor.log'
-    ServiceProvider.new.monitor
-  end
-  
-  def stop_runners_daemon_mode
-    with_daemon_gem do
-      Daemons.run_proc "dtr_runners", :ARGV => ['stop'] 
-    end
-  end
-  
-  def start_runners_daemon_mode
-    with_daemon_gem do
-      FileUtils.rm_rf('dtr_runners.output')
-      pwd = Dir.pwd
-      Daemons.run_proc "dtr_runners", :ARGV => ['start'], :backtrace => true  do
-        Dir.chdir(pwd) do
-          start_runners
-        end
-      end
-    end
-  end
-  
-  def start_server_daemon_mode
-    with_daemon_gem do
-      Daemons.run_proc "dtr_server", :ARGV => ['start'] do
-        start_server
-      end
-    end
-    sleep(2)
-  end
-  
-  def stop_server_daemon_mode
-    with_daemon_gem do
-      Daemons.run_proc "dtr_server", :ARGV => ['stop']
-    end
-  end
-  
-  def with_daemon_gem
-    require "rubygems"
-    begin
-      require "daemons"
-    rescue LoadError
-      raise "The daemons gem must be installed"
-    end
-    yield
-  end
-  
-  def clean_server
-    require 'dtr/service_provider'
-    ServiceProvider.new.clear_workspace
-  end
-  
-  module_function :start_server, :start_runners, :launch_runners, :lib_path, :broadcast_list=, :runners, :with_daemon_gem, :start_runners_daemon_mode, :stop_runners_daemon_mode, :start_server_daemon_mode, :stop_server_daemon_mode, :monitor, :port=, :clean_server
+  extend Facade
 end

data/lib/dtr/agent.rb

@@ -0,0 +1,38 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared'
+require 'dtr/agent/working_env_ext'
+require 'dtr/agent/sync_codebase'
+
+require 'dtr/agent/sync_logger'
+require 'dtr/agent/brain'
+require 'dtr/agent/worker'
+require 'dtr/agent/test_unit'
+require 'dtr/agent/herald'
+require 'dtr/agent/test_case'
+require 'dtr/agent/runner'
+
+module DTR
+  module Agent
+    def start(runner_names=["Distributed Test Runner"], agent_env_setup_cmd=nil)
+      DTR.logger('dtr_agent.log')
+      Brain.new(runner_names, agent_env_setup_cmd).hypnotize
+    end
+    
+    module_function :start
+  end
+  WorkingEnv.send(:include, Agent::WorkingEnvExt)
+  WorkingEnv.send(:include, SyncCodebase::WorkingEnvExt)
+end

data/lib/dtr/agent/brain.rb

@@ -0,0 +1,57 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Agent
+
+    class Brain
+      include Adapter::Follower
+      
+      def initialize(runner_names, agent_env_setup_cmd)
+        raise 'No runner? What can I do for you?' if runner_names.blank?
+        @runner_names = runner_names
+        @agent_env_setup_cmd = agent_env_setup_cmd
+        DTR.info {""}
+        DTR.info {"--------------------beautiful line--------------------------"}
+        DTR.info {"=> Agent environment setup command: #{@agent_env_setup_cmd}"}
+        DTR.info {"=> Runner names: #{@runner_names.join(', ')}"}
+        DTR.info {"=> Broadcast list: #{DTR.configuration.broadcast_list.inspect}"}
+        DTR.info {"=> Listening port: #{DTR.configuration.agent_listen_port}"}
+        DTR.info {"=> Group: #{DTR.configuration.group}"}
+      end
+
+      def hypnotize
+        loop do
+          if wakeup?
+            DTR.info {"Agent brain wakes up"}
+            work(DTR.fork_process { Worker.new(@runner_names, @agent_env_setup_cmd).launch })
+            DTR.info {"Agent brain is going to sleep"}
+          end
+        end
+      rescue Interrupt, SystemExit, SignalException
+      ensure
+        relax
+      end
+
+      def work(worker)
+        until sleep?
+          #keep worker working :D
+        end
+      ensure
+        DTR.info {"Killing worker"}
+        DTR.kill_process worker
+      end
+    end
+  end
+end

data/lib/dtr/agent/herald.rb

@@ -0,0 +1,60 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+
+  module Agent
+    class Herald
+      class WorkingEnvError < StandardError
+      end
+
+      include Service::WorkingEnv
+      include Service::Agent
+
+      def initialize(working_env_key, agent_env_setup_cmd, runners)
+        @working_env_key = working_env_key
+        @agent_env_setup_cmd = agent_env_setup_cmd
+        @runners = runners
+        @env_store = EnvStore.new
+        start_service
+        start_off
+      ensure
+        stop_service
+      end
+
+      def start_off
+        DTR.info {"=> Herald starts off..."}
+        provide_agent_info(@agent_env_setup_cmd, @runners)
+
+        @env_store[@working_env_key] = fetch_working_env
+      rescue WorkingEnvError
+        DTR.error $!.message
+        exit(-1)
+      rescue
+        DTR.error $!.message
+        DTR.error $!.backtrace.join("\n")
+        exit(-1)
+      end
+
+      def fetch_working_env
+        working_env = lookup_working_env
+        DTR.info {"=> Got working environment created at #{working_env[:created_at]} by #{working_env[:host]}"}
+
+        raise WorkingEnvError.new("No test files need to load?(working env: #{working_env})") if working_env[:files].blank?
+        raise WorkingEnvError.new('Setup working environment failed, no runner started.') unless working_env.setup_env(@agent_env_setup_cmd)
+        working_env
+      end
+    end
+  end
+end