dtr 0.0.4 → 1.0.0

data/lib/dtr/master.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared'
+
+module DTR
+  module Master
+    def with_dtr_master(&block)
+      if defined?(ActiveRecord::Base)
+        ActiveRecord::Base.clear_active_connections! rescue nil
+      end
+
+      DTR.info {""}
+      DTR.info {"--------------------beautiful line--------------------------"}
+      DTR.info {"Master process started at #{Time.now}"}
+
+      DTR.configuration.with_rinda_server do
+        provide_working_env WorkingEnv.new
+        with_wakeup_agents(&block)
+      end
+    end
+
+    include Adapter::Master
+    include Service::WorkingEnv
+    include SyncCodebase::MasterExt
+  end
+
+  Configuration.send(:include, SyncLogger::Provider)
+end

data/lib/dtr/monitor.rb

@@ -0,0 +1,95 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/master'
+
+module DTR
+  
+  # Monitor provides a way to monitor agent/master process working status.
+  class Monitor
+
+    class AgentsMonitor
+      include Adapter::Master
+
+      def start
+        DTR.fork_process do
+          monitor
+        end
+      end
+      def monitor
+        DTR.configuration.with_rinda_server do
+          with_wakeup_agents do
+            begin
+              sleep
+            rescue Interrupt
+            end
+          end
+        end
+      end
+    end
+
+    class MasterMonitor
+      include Adapter::Follower
+      include Service::Agent
+
+      def start
+        DTR.fork_process do
+          begin
+            loop do
+              monitor
+            end
+          rescue Errno::EADDRINUSE
+            puts "There is DTR agent started on this machine."
+            puts "Shutdown it for monitoring working DTR Master info."
+          rescue Interrupt
+          ensure
+            relax
+          end
+        end
+      end
+
+      def monitor
+        msg, from_host, group = listen
+        unless ["127.0.0.1:#{DTR.configuration.rinda_server_port}"].include?(from_host)
+          puts "Master process message from #{from_host}: #{msg} for group #{group}"
+          ip, port = from_host.split(':')
+          with_configuration(ip, port) do
+            start_service
+            puts "Agents working for #{from_host}: "
+            puts all_agents_info.collect{|i| "  #{i.strip}"}.join("\n")
+          end
+        end
+      end
+
+      def with_configuration(ip, port)
+        my_port = DTR.configuration.rinda_server_port
+        my_broadcast_list = DTR.configuration.broadcast_list
+        DTR.configuration.rinda_server_port = port.to_i
+        DTR.configuration.broadcast_list = [ip]
+        yield
+      ensure
+        DTR.configuration.broadcast_list = my_broadcast_list
+        DTR.configuration.rinda_server_port = my_port
+      end
+    end
+
+    def start
+      MasterMonitor.new.start
+      AgentsMonitor.new.start
+      puts "Monitor process started at #{Time.now}"
+      Process.waitall
+    rescue Interrupt
+    end
+  end
+end

data/lib/dtr/raketasks.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2007-2008 Li Xiao
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
 # 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,21 +13,79 @@
 # limitations under the License.
 
 require "rubygems"
-require 'dtr'
+require 'rake'
 require 'rake/testtask'
+require 'rake/tasklib'
+
+require 'dtr'
+require 'dtr/shared/ruby_ext'
+require 'dtr/shared/utils'
+require 'dtr/shared/sync_codebase/package'
 
 module DTR
-  class MPTask < Rake::TestTask
-    attr_accessor :processes, :runner_options, :start_server
-    
+  # Create tasks that run a set of tests with DTR injected.
+  # The TestTask will create the following targets:
+  #
+  # [<b>:dtr</b>]: 
+  #   Create a task that runs a set of tests by DTR master.
+  #
+  # [DTR::PackageTask]: 
+  #   Create a packaging task that will package the project into
+  #   distributable files for running test on remote machine.
+  #   All test files should be included.
+  #
+  # Example:
+  #   require 'dtr/raketasks'
+  #
+  #   DTR::TestTask.new do |t|
+  #     t.libs << "test"
+  #     t.test_files = FileList['test/test*.rb']
+  #     t.verbose = true
+  #     t.processes = 1 # default is 1
+  #     t.package_files.include("lib/**/*") # default is FileList["**/*"]
+  #     t.package_files.include("test/**/*")
+  #   end
+  #
+  # This task inherits from Rake::TestTask, and adds 2 DTR specific 
+  # options: processes and package_files.
+  #
+  class TestTask < Rake::TestTask
+
+    #
+    # The option processes is used to start an DTR agent in same directory
+    # with master process. The number of processes is the size of runners
+    # launched by agent for running tests. If processes is set to 0, 
+    # then there is no agent started locally.
+    # Default is 1.
+    attr_accessor :processes
+
+    # List of files to be included in the package for running tests on
+    # remote agent.
+    # The agent, which starts in same directory on same machine with 
+    # master process, would skip copying codebase.
+    # The default package files is Rake::FileList["**/*"].
+    attr_accessor :package_files
+
+    def initialize(name=:dtr)
+      @processes = 1
+      @package_files = Rake::FileList.new
+      super(name)
+    end
+
     def define
+      PackageTask.new do |p|
+        p.package_files = package_files
+        if p.package_files.empty?
+          p.package_files.include("**/*")
+        end
+      end
+
       @libs.unshift DTR.lib_path
       lib_path = @libs.join(File::PATH_SEPARATOR)
 
-      desc "Run tests" + (@name==:test ? "" : " for #{@name}")
+      desc "Run tests with DTR injected"
       task @name do
-        DTR.start_server_daemon_mode if start_server?
-        start_runners
+        @agent = start_agent
         run_code = ''
         begin
           RakeFileUtils.verbose(@verbose) do
@@ -41,29 +99,104 @@ module DTR
               " #{option_list}"
           end
         ensure
-          DTR.stop_runners_daemon_mode rescue nil
-          if start_server?
-            DTR.stop_server_daemon_mode rescue nil
+          if defined?(@agent)
+            DTR.kill_process @agent
           end
         end
       end
       self
     end
-    
-    def processes
-      @processes ? @processes.to_i : 2
-    end
-    
-    def start_server?
-      defined?(@start_server) ? @start_server : true
-    end
-    
+
     private
-    def start_runners
+    def start_agent
       return if self.processes.to_i <= 0
       runner_names = []
       self.processes.to_i.times {|i| runner_names << "runner#{i}"}
-      %x[dtr -r #{runner_names.join(',')} -D #{runner_options}]
+      
+      DTR.fork_process do
+        DTR_AGENT_OPTIONS[:runners] = runner_names if DTR_AGENT_OPTIONS[:runners].empty?
+        DTR.start_agent
+      end
+    end
+  end
+
+  # Create a packaging task that will package the project into
+  # distributable files for running test on remote machine.
+  # It uses zip and unzip to package and unpackage files.
+  # All test files should be included.
+  #
+  # The PackageTask will create the following targets:
+  #
+  # [<b>:dtr_package</b>]
+  #   Create all the requested package files.
+  #
+  # [<b>:dtr_clobber_package</b>]
+  #   Delete all the package files. This target is automatically
+  #   added to the main clobber target.
+  #
+  # [<b>:dtr_repackage</b>]
+  #   Rebuild the package files from scratch, even if they are not out
+  #   of date.
+  #
+  # Example:
+  #
+  #   DTR::PackageTask.new do |p|
+  #     p.package_files.include("lib/**/*.rb")
+  #     p.package_files.include("test/**/*.rb")
+  #   end
+  #
+  class PackageTask < Rake::TaskLib
+    include SyncCodebase::Package
+    # List of files to be included in the package.
+    attr_accessor :package_files
+
+    # Create a Package Task with the given name and version. 
+    def initialize
+      @package_files = Rake::FileList.new
+      yield self if block_given?
+      define
+    end
+
+    # Create the tasks defined by this task library.
+    def define
+      desc "Build packages for dtr task"
+      task :dtr_package
+
+      desc "Force a rebuild of the package files for dtr task"
+      task :dtr_repackage => [:dtr_clobber_package, :dtr_package]
+
+      desc "Remove package for dtr task" 
+      task :dtr_clobber_package do
+        rm_r package_dir rescue nil
+      end
+
+      file, flag = package_file, 'j'
+      task :dtr_package => ["#{package_dir}/#{file}"]
+
+      file "#{package_dir}/#{file}" => [package_dir_path] do
+        chdir(package_dir) do
+          do_work(package_cmd)
+        end
+      end
+
+      directory package_dir
+
+      file package_dir_path do
+        mkdir_p package_dir rescue nil
+        @package_files.exclude(package_dir)
+        @package_files.each do |fn|
+          f = File.join(package_dir_path, fn)
+          fdir = File.dirname(f)
+          mkdir_p(fdir) if !File.exist?(fdir)
+          if File.directory?(fn)
+            mkdir_p(f)
+          else
+            rm_f f
+            safe_ln(fn, f)
+          end
+        end
+      end
+      self
     end
   end
 end

data/lib/dtr/shared.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+require 'dtr/shared/ruby_ext'
+require 'dtr/shared/utils'
+require 'dtr/shared/message_decorator'
+require 'dtr/shared/working_env'
+require 'dtr/shared/service'
+require 'dtr/shared/configuration'
+require 'dtr/shared/sync_logger'
+require 'dtr/shared/adapter'
+require 'dtr/shared/sync_codebase'

data/lib/dtr/shared/adapter.rb

@@ -0,0 +1,115 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+
+module DTR
+  module Adapter
+    WAKEUP_MESSAGE = 'wakeup'
+    SLEEP_MESSAGE = 'sleep'
+
+    module Follower
+      def wakeup?
+        msg, host, group = listen
+        if group == DTR.configuration.group && msg == Adapter::WAKEUP_MESSAGE
+          ip, port = host.split(':')
+          DTR.configuration.rinda_server_port = port.to_i
+          DTR.configuration.broadcast_list = [ip]
+          @wakeup_for_host = host
+          true
+        end
+      end
+
+      def sleep?
+        return true unless defined?(@wakeup_for_host)
+        Timeout.timeout(DTR.configuration.follower_listen_heartbeat_timeout) do
+          until (msg, host = listen) && host == @wakeup_for_host; end
+          msg == Adapter::SLEEP_MESSAGE
+        end
+      rescue Timeout::Error => e
+        DTR.info {"Timeout while listening command"}
+        true
+      end
+
+      def relax
+        if defined?(@soc) && @soc
+          @soc.close rescue nil
+        end
+      end
+
+      private
+      def listen
+        unless defined?(@soc)
+          @soc = UDPSocket.open
+          @soc.bind('', DTR.configuration.agent_listen_port)
+          DTR.info("DTR Agent is listening on port #{DTR.configuration.agent_listen_port}")
+        end
+        message, client_address = @soc.recvfrom(400)
+        cmd, port, group = message.split
+
+        hostname = client_address[2]
+        host_ip = client_address[3]
+        DTR.info {"Received: #{cmd} for group #{group} from #{hostname}(#{host_ip}):#{port}"}
+        [cmd, "#{host_ip}:#{port}", group]
+      end
+    end
+
+    module Master
+      def hypnotize_agents
+        yell_agents("#{Adapter::SLEEP_MESSAGE} #{DTR.configuration.rinda_server_port}")
+      end
+
+      def with_wakeup_agents(&block)
+        heartbeat = Thread.new do
+          loop do
+            do_wakeup_agents
+            sleep(DTR.configuration.master_heartbeat_interval)
+          end
+        end
+        #heartbeat thread should have high priority for agent is listening
+        heartbeat.priority = Thread.current.priority + 10
+        block.call
+      ensure
+        #kill heartbeat first, so that agents wouldn't be wakeup after hypnotized
+        Thread.kill heartbeat rescue nil
+        hypnotize_agents rescue nil
+      end
+
+      def do_wakeup_agents
+        yell_agents("#{Adapter::WAKEUP_MESSAGE} #{DTR.configuration.rinda_server_port} #{DTR.configuration.group}")
+      end
+
+      private
+      def yell_agents(msg)
+        DTR.info {"yell agents #{msg}: #{DTR.configuration.broadcast_list.inspect}"}
+        DTR.configuration.broadcast_list.each do |it|
+          broadcast(it, msg)
+        end
+      end
+
+      def broadcast(it, msg)
+        soc = UDPSocket.open
+        begin
+          soc.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, true)
+          DTR.debug {"broadcast sending #{msg} to #{it}"}
+          soc.send(msg, 0, it, DTR.configuration.agent_listen_port)
+        rescue
+          nil
+        ensure
+          soc.close
+        end
+      end
+    end
+  end
+end