xli-dtr 0.0.5 → 1.0.0

data/lib/dtr/raketasks.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,22 +13,77 @@
 # limitations under the License.
 
 require "rubygems"
-require 'dtr'
-require 'dtr/shared/sync_codebase/package'
-require 'dtr/shared/ruby_ext'
 require 'rake'
 require 'rake/testtask'
 require 'rake/tasklib'
 
+require 'dtr'
+require 'dtr/shared/ruby_ext'
+require 'dtr/shared/utils'
+require 'dtr/shared/sync_codebase/package'
+
 module DTR
+  # Create tasks that run a set of tests with DTR injected.
+  # The TestTask will create the following targets:
+  #
+  # [<b>:dtr</b>]: 
+  #   Create a task that runs a set of tests by DTR master.
+  #
+  # [DTR::PackageTask]: 
+  #   Create a packaging task that will package the project into
+  #   distributable files for running test on remote machine.
+  #   All test files should be included.
+  #
+  # Example:
+  #   require 'dtr/raketasks'
+  #
+  #   DTR::TestTask.new do |t|
+  #     t.libs << "test"
+  #     t.test_files = FileList['test/test*.rb']
+  #     t.verbose = true
+  #     t.processes = 1 # default is 1
+  #     t.package_files.include("lib/**/*") # default is FileList["**/*"]
+  #     t.package_files.include("test/**/*")
+  #   end
+  #
+  # This task inherits from Rake::TestTask, and adds 2 DTR specific 
+  # options: processes and package_files.
+  #
   class TestTask < Rake::TestTask
+
+    #
+    # The option processes is used to start an DTR agent in same directory
+    # with master process. The number of processes is the size of runners
+    # launched by agent for running tests. If processes is set to 0, 
+    # then there is no agent started locally.
+    # Default is 1.
     attr_accessor :processes
-    
+
+    # List of files to be included in the package for running tests on
+    # remote agent.
+    # The agent, which starts in same directory on same machine with 
+    # master process, would skip copying codebase.
+    # The default package files is Rake::FileList["**/*"].
+    attr_accessor :package_files
+
+    def initialize(name=:dtr)
+      @processes = 1
+      @package_files = Rake::FileList.new
+      super(name)
+    end
+
     def define
+      PackageTask.new do |p|
+        p.package_files = package_files
+        if p.package_files.empty?
+          p.package_files.include("**/*")
+        end
+      end
+
       @libs.unshift DTR.lib_path
       lib_path = @libs.join(File::PATH_SEPARATOR)
 
-      desc "Run tests" + (@name==:test ? "" : " for #{@name}")
+      desc "Run tests with DTR injected"
       task @name do
         @agent = start_agent
         run_code = ''
@@ -45,43 +100,59 @@ module DTR
           end
         ensure
           if defined?(@agent)
-            Process.kill 'TERM', @agent rescue nil
+            DTR.kill_process @agent
           end
         end
       end
       self
     end
-    
-    def processes
-      @processes ? @processes.to_i : 2
-    end
-    
+
     private
     def start_agent
       return if self.processes.to_i <= 0
       runner_names = []
       self.processes.to_i.times {|i| runner_names << "runner#{i}"}
       
-      Process.fork do
+      DTR.fork_process do
         DTR_AGENT_OPTIONS[:runners] = runner_names if DTR_AGENT_OPTIONS[:runners].empty?
         DTR.start_agent
       end
     end
   end
 
-  # The following task is copied & modified from 'rake/packagetask'
+  # Create a packaging task that will package the project into
+  # distributable files for running test on remote machine.
+  # It uses zip and unzip to package and unpackage files.
+  # All test files should be included.
+  #
+  # The PackageTask will create the following targets:
+  #
+  # [<b>:dtr_package</b>]
+  #   Create all the requested package files.
+  #
+  # [<b>:dtr_clobber_package</b>]
+  #   Delete all the package files. This target is automatically
+  #   added to the main clobber target.
+  #
+  # [<b>:dtr_repackage</b>]
+  #   Rebuild the package files from scratch, even if they are not out
+  #   of date.
+  #
+  # Example:
+  #
+  #   DTR::PackageTask.new do |p|
+  #     p.package_files.include("lib/**/*.rb")
+  #     p.package_files.include("test/**/*.rb")
+  #   end
+  #
   class PackageTask < Rake::TaskLib
     include SyncCodebase::Package
     # List of files to be included in the package.
     attr_accessor :package_files
 
-    # Tar command for gzipped or bzip2ed archives.  The default is 'tar'.
-    attr_accessor :tar_command
-
     # Create a Package Task with the given name and version. 
     def initialize
       @package_files = Rake::FileList.new
-      @tar_command = 'tar'
       yield self if block_given?
       define
     end
@@ -104,7 +175,7 @@ module DTR
 
       file "#{package_dir}/#{file}" => [package_dir_path] do
         chdir(package_dir) do
-          sh %{#{@tar_command} #{flag}cvf #{file} #{package_name}}
+          do_work(package_cmd)
         end
       end
 
@@ -112,6 +183,7 @@ module DTR
 
       file package_dir_path do
         mkdir_p package_dir rescue nil
+        @package_files.exclude(package_dir)
         @package_files.each do |fn|
           f = File.join(package_dir_path, fn)
           fdir = File.dirname(f)

data/lib/dtr/shared/adapter.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,17 +16,16 @@ require 'timeout'
 
 module DTR
   module Adapter
-    AGENT_PORT = 7788
-
     WAKEUP_MESSAGE = 'wakeup'
     SLEEP_MESSAGE = 'sleep'
 
     module Follower
       def wakeup?
-        msg, host = listen
-        if msg == Adapter::WAKEUP_MESSAGE
-          port = host.split(':').last.to_i
-          DTR.configuration.rinda_server_port = port
+        msg, host, group = listen
+        if group == DTR.configuration.group && msg == Adapter::WAKEUP_MESSAGE
+          ip, port = host.split(':')
+          DTR.configuration.rinda_server_port = port.to_i
+          DTR.configuration.broadcast_list = [ip]
           @wakeup_for_host = host
           true
         end
@@ -34,40 +33,48 @@ module DTR
 
       def sleep?
         return true unless defined?(@wakeup_for_host)
-        msg, host = Timeout.timeout(DTR.configuration.follower_listen_sleep_timeout) do
-          loop do
-            msg, host = listen
-            break if host == @wakeup_for_host
-          end
-          [msg, host]
+        Timeout.timeout(DTR.configuration.follower_listen_heartbeat_timeout) do
+          until (msg, host = listen) && host == @wakeup_for_host; end
+          msg == Adapter::SLEEP_MESSAGE
         end
-        DTR.info "Received: #{msg} from #{host}"
-        msg == Adapter::SLEEP_MESSAGE
       rescue Timeout::Error => e
-        DTR.info "Timeout while listening command"
+        DTR.info {"Timeout while listening command"}
         true
       end
 
+      def relax
+        if defined?(@soc) && @soc
+          @soc.close rescue nil
+        end
+      end
+
       private
       def listen
         unless defined?(@soc)
           @soc = UDPSocket.open
-          @soc.bind('', Adapter::AGENT_PORT)
+          @soc.bind('', DTR.configuration.agent_listen_port)
+          DTR.info("DTR Agent is listening on port #{DTR.configuration.agent_listen_port}")
         end
-        @soc.recv(1024).split
+        message, client_address = @soc.recvfrom(400)
+        cmd, port, group = message.split
+
+        hostname = client_address[2]
+        host_ip = client_address[3]
+        DTR.info {"Received: #{cmd} for group #{group} from #{hostname}(#{host_ip}):#{port}"}
+        [cmd, "#{host_ip}:#{port}", group]
       end
     end
 
     module Master
       def hypnotize_agents
-        yell_agents("#{Adapter::SLEEP_MESSAGE} #{host}")
+        yell_agents("#{Adapter::SLEEP_MESSAGE} #{DTR.configuration.rinda_server_port}")
       end
 
       def with_wakeup_agents(&block)
         heartbeat = Thread.new do
           loop do
             do_wakeup_agents
-            sleep(DTR.configuration.master_yell_interval)
+            sleep(DTR.configuration.master_heartbeat_interval)
           end
         end
         #heartbeat thread should have high priority for agent is listening
@@ -80,7 +87,7 @@ module DTR
       end
 
       def do_wakeup_agents
-        yell_agents("#{Adapter::WAKEUP_MESSAGE} #{host}")
+        yell_agents("#{Adapter::WAKEUP_MESSAGE} #{DTR.configuration.rinda_server_port} #{DTR.configuration.group}")
       end
 
       private
@@ -91,16 +98,12 @@ module DTR
         end
       end
 
-      def host
-        "#{Socket.gethostname}:#{DTR.configuration.rinda_server_port}"
-      end
-
       def broadcast(it, msg)
         soc = UDPSocket.open
         begin
           soc.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, true)
           DTR.debug {"broadcast sending #{msg} to #{it}"}
-          soc.send(msg, 0, it, Adapter::AGENT_PORT)
+          soc.send(msg, 0, it, DTR.configuration.agent_listen_port)
         rescue
           nil
         ensure

data/lib/dtr/shared/configuration.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 require 'singleton'
-require 'rinda/rinda'
+require 'rinda/ring'
 require 'rinda/tuplespace'
 
 module DTR
@@ -24,31 +24,59 @@ module DTR
 
   module_function :configuration
 
+  # DTR Configuration includes: 
+  #   For both master & agent process:
+  #     * broadcast_list: broadcast ip list for looking for dtr services(agent and DTR master rinda server).
+  #     * group: grouping agents for specific usage, master process should specify same group for catching agents to work
+  #   For master process:
+  #     * rinda_server_port: DTR would automatically detect unused port for master rinda server. The default port is 3344.
+  #     * master_heartbeat_interval: master process heartbeat for keeping agents working. The default is 10 sec.
+  #   For agent process:
+  #     * agent_listen_port: agent listen master process command port, the default port is 7788.
+  #     * follower_listen_heartbeat_timeout: agents would be going to sleep if listening master heartbeat timeout. The default is 15 sec.
+  #
+  # All configurations except rinda_server_port would be stored into pstore file .dtr_env_pstore in the DTR process launching directory.
+  #
   class Configuration
     include Singleton
     include Service::Rinda
 
-    attr_accessor :broadcast_list, :rinda_server_port, :master_yell_interval, :follower_listen_sleep_timeout
+    attr_accessor :broadcast_list, :rinda_server_port, :agent_listen_port, :master_heartbeat_interval, :follower_listen_heartbeat_timeout
+    attr_reader :group
 
     def initialize
+      load
+    end
+
+    def load
       store = EnvStore.new
+      # always have 'localhost' in broadcast_list, for our master process would start rinda server locally,
+      # and dtr should work well on local machine when the machine leaves dtr grid network environment.
       @broadcast_list = ['localhost'].concat(store[:broadcast_list] || []).uniq
-      @rinda_server_port = store[:port] || 3344
-      @master_yell_interval = store[:master_yell_interval] || 10
-      @follower_listen_sleep_timeout =  store[:follower_listen_sleep_timeout] || 15
+      @rinda_server_port = 3344
+      @agent_listen_port = store[:agent_listen_port] || 7788
+      @master_heartbeat_interval = store[:master_heartbeat_interval] || 10
+      @follower_listen_heartbeat_timeout =  store[:follower_listen_heartbeat_timeout] || 15
+      @group = store[:group]
     end
 
     def save
       store = EnvStore.new
       store[:broadcast_list] = @broadcast_list
-      store[:port] = @rinda_server_port
-      store[:master_yell_interval] = @master_yell_interval
-      store[:follower_listen_sleep_timeout] = @follower_listen_sleep_timeout
+      store[:agent_listen_port] = @agent_listen_port
+      store[:master_heartbeat_interval] = @master_heartbeat_interval
+      store[:follower_listen_heartbeat_timeout] = @follower_listen_heartbeat_timeout
+      store[:group] = @group
+    end
+
+    def group=(group)
+      @group = group.blank? ? nil : group.gsub(' ', '_')
     end
 
     def with_rinda_server(&block)
+      DTR.do_println("Booting DTR service")
       start_service
-      DTR.info '-- Booting DTR Rinda server...'
+      DTR.info {'-- Booting DTR Rinda server'}
       loop do
         begin
           Rinda::RingServer.new Rinda::TupleSpace.new, @rinda_server_port
@@ -57,7 +85,7 @@ module DTR
           @rinda_server_port += 1
         end
       end
-      DTR.info "-- DTR Rinda server started on port #{@rinda_server_port}"
+      DTR.info {"-- DTR Rinda server started on port #{@rinda_server_port}"}
       block.call
     ensure
       stop_service rescue nil

data/lib/dtr/shared/message_decorator.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/dtr/shared/ruby_ext.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,30 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-class Null
-  class << self
-    def instance(overrides = {})
-      self.new.define_overrides(overrides)
-    end  
-  end
-
-  #override Object#id for removing the warning
-  def id
-    nil
-  end
-
-  def method_missing(sym, *args, &block)
-    nil
-  end
-
-  def define_overrides(overrides)
-    overrides.each_pair do |key, value|
-      (class << self; self; end;).send(:define_method, key, lambda { value })
-    end
-    self
-  end
-end
-
 # from activesupport-2.1.1
 class Object
   def blank?

data/lib/dtr/shared/service/agent.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,6 +28,10 @@ module DTR
 }
         lookup_ring.write [:agent, agent]
       end
+
+      def all_agents_info
+        lookup_ring.read_all([:agent, nil]).collect{|t|t[1]}
+      end
     end
   end
 end

data/lib/dtr/shared/service/file.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/dtr/shared/service/rinda.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,8 +20,16 @@ module DTR
     module Rinda
 
       def start_service
-        DTR.info "-- Start drb service..."
-        DRb.start_service
+        DTR.info {"-- Start drb service..."}
+        #todo need figure why agents couldn't be killed directly when start drb service by DRb.start_service
+        if ENV['DTR_ENV'] == 'test'
+          DRb.start_service("druby://#{Socket.gethostname}:0")
+        else
+          DRb.start_service
+        end
+      rescue
+        # for ruby 1.8.7 need specify uri
+        DRb.start_service("druby://#{Socket.gethostname}:0")
       end
 
       def stop_service

data/lib/dtr/shared/service/runner.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,14 +20,15 @@ module DTR
       include Rinda
 
       def provide_runner(runner)
-        renewer = ::Rinda::SimpleRenewer.new
-        tuple = [:name, 'DTR::Runner'.to_sym, runner, "DTR remote runner #{Process.pid}-#{runner.name}"]
-        lookup_ring.write(tuple, renewer)
+        tuple = ['DTR::Runner'.to_sym, runner, "DTR remote runner #{Process.pid}-#{runner.name}"]
+        #expires after 1 sec for we don't need runner service anymore if there is no one is waiting for taking it
+        lookup_ring.write(tuple, 1)
       end
 
       def lookup_runner
-        lookup(:take, [:name, 'DTR::Runner'.to_sym, nil, nil])[2]
+        lookup(:take, ['DTR::Runner'.to_sym, nil, nil])[1]
       end
+
     end
   end
 end

data/lib/dtr/shared/service/working_env.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/dtr/shared/service.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/dtr/shared/sync_codebase/{codebase.rb → copiable_package.rb}

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,19 +14,27 @@
 
 module DTR
   module SyncCodebase
-    class Codebase
+    class CopiablePackage
       include DRbUndumped
       include Package
 
       CHUNK_SIZE = 1024*1024
 
-      def write(remote_io)
-        File.open(File.join(package_dir, package_file), "rb") do |f|
-          while chunk = f.read(CHUNK_SIZE) and chunk.length > 0
+      def initialize
+        raise "Package(#{codebase_package}) doesn't exist!" unless File.exist?(codebase_package)
+      end
+
+      def copy_into(remote_io)
+        File.open(codebase_package, "rb") do |f|
+          while (chunk = f.read(CHUNK_SIZE) || '') && chunk.length > 0
             remote_io.write(chunk)
           end
         end
       end
+
+      def codebase_package
+        File.join(package_dir, package_file)
+      end
     end
   end
 end

data/lib/dtr/shared/sync_codebase/master_ext.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'test/unit/testresult'
-
 module DTR
   module SyncCodebase
+    # Inject synchronizing codebase ability into Master#with_dtr_master
+    # Packaging codebase by rake task dtr_repackage (See DTR::PackageTask)
     module MasterExt
       include Service::File
 
@@ -25,22 +25,10 @@ module DTR
 
       def with_dtr_master_with_sync_codebase(&block)
         with_dtr_master_without_sync_codebase do
-          unless Cmd.execute('rake dtr_repackage --trace')
-            $stderr.puts %{
-Execute dtr_repackage rake task failed, see log for details.
-For running DTR test task, you must define a DTR::PackageTask task in your rakefile for DTR need package and synchronize your codebase within grid.
-Example:
-  require 'dtr/raketasks'
-  DTR::PackageTask.new do |p|
-    p.package_files.include("**/*")
-    p.package_files.exclude("tmp")
-    p.package_files.exclude("log")
-  end
-}
-            return Test::Unit::TestResult.new
-          end
+          DTR.do_println("Packaging codebase")
+          raise 'Packaging for dtr task failed, see log for details!' unless Cmd.execute('rake dtr_repackage --trace')
           begin
-            provide_file Codebase.new
+            provide_file CopiablePackage.new
             block.call
           ensure
             Cmd.execute('rake dtr_clobber_package --trace')

data/lib/dtr/shared/sync_codebase/package.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,12 +28,26 @@ module DTR
       end
 
       def package_file
-        "#{package_name}.tar.bz2"
+        "#{package_name}.zip"
       end
 
       def package_copy_file
         "copy_#{package_file}"
       end
+
+      def unpackage_cmd
+        "unzip #{package_copy_file}"
+      end
+
+      def package_cmd
+        "zip -r #{package_file} #{package_name}"
+      end
+
+      def do_work(cmd)
+        unless Cmd.execute(cmd)
+          raise "Execute command: #{cmd} failed. See log for details."
+        end
+      end
     end
   end
 end

data/lib/dtr/shared/sync_codebase/sync_service.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,21 +18,16 @@ module DTR
       include Package
       include Service::File
       def sync_codebase
-        DTR.info("start sync codebase, clean #{File.join(Dir.pwd, package_name)}")
+        DTR.info("Start sync codebase, clean #{File.join(Dir.pwd, package_name)}")
         FileUtils.rm_rf(File.join(Dir.pwd, package_name))
 
-        DTR.info("lookup codebase file")
-        codebase = lookup_file
-        DTR.info("receiving codebase: #{package_copy_file}")
+        DTR.info("Lookup codebase package file")
+        package = lookup_file
+        DTR.info("Receiving package file and writing to #{package_copy_file}")
         File.open(package_copy_file, 'w') do |f|
-          codebase.write(f)
-        end
-        unless File.exists?(package_copy_file)
-          raise "#{package_copy_file} does not exist, sync codebase failed."
-        end
-        unless Cmd.execute("tar -xjf #{package_copy_file}")
-          raise "Extracting #{package_copy_file} by 'tar' failed."
+          package.copy_into(f)
         end
+        do_work(unpackage_cmd)
         DTR.info("sync codebase finished, clean #{package_copy_file}")
         FileUtils.rm_f(package_copy_file)
       end