xli-dtr 0.0.4 → 0.0.5

data/lib/dtr/shared/adapter.rb

@@ -0,0 +1,112 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+
+module DTR
+  module Adapter
+    AGENT_PORT = 7788
+
+    WAKEUP_MESSAGE = 'wakeup'
+    SLEEP_MESSAGE = 'sleep'
+
+    module Follower
+      def wakeup?
+        msg, host = listen
+        if msg == Adapter::WAKEUP_MESSAGE
+          port = host.split(':').last.to_i
+          DTR.configuration.rinda_server_port = port
+          @wakeup_for_host = host
+          true
+        end
+      end
+
+      def sleep?
+        return true unless defined?(@wakeup_for_host)
+        msg, host = Timeout.timeout(DTR.configuration.follower_listen_sleep_timeout) do
+          loop do
+            msg, host = listen
+            break if host == @wakeup_for_host
+          end
+          [msg, host]
+        end
+        DTR.info "Received: #{msg} from #{host}"
+        msg == Adapter::SLEEP_MESSAGE
+      rescue Timeout::Error => e
+        DTR.info "Timeout while listening command"
+        true
+      end
+
+      private
+      def listen
+        unless defined?(@soc)
+          @soc = UDPSocket.open
+          @soc.bind('', Adapter::AGENT_PORT)
+        end
+        @soc.recv(1024).split
+      end
+    end
+
+    module Master
+      def hypnotize_agents
+        yell_agents("#{Adapter::SLEEP_MESSAGE} #{host}")
+      end
+
+      def with_wakeup_agents(&block)
+        heartbeat = Thread.new do
+          loop do
+            do_wakeup_agents
+            sleep(DTR.configuration.master_yell_interval)
+          end
+        end
+        #heartbeat thread should have high priority for agent is listening
+        heartbeat.priority = Thread.current.priority + 10
+        block.call
+      ensure
+        #kill heartbeat first, so that agents wouldn't be wakeup after hypnotized
+        Thread.kill heartbeat rescue nil
+        hypnotize_agents rescue nil
+      end
+
+      def do_wakeup_agents
+        yell_agents("#{Adapter::WAKEUP_MESSAGE} #{host}")
+      end
+
+      private
+      def yell_agents(msg)
+        DTR.info {"yell agents #{msg}: #{DTR.configuration.broadcast_list.inspect}"}
+        DTR.configuration.broadcast_list.each do |it|
+          broadcast(it, msg)
+        end
+      end
+
+      def host
+        "#{Socket.gethostname}:#{DTR.configuration.rinda_server_port}"
+      end
+
+      def broadcast(it, msg)
+        soc = UDPSocket.open
+        begin
+          soc.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, true)
+          DTR.debug {"broadcast sending #{msg} to #{it}"}
+          soc.send(msg, 0, it, Adapter::AGENT_PORT)
+        rescue
+          nil
+        ensure
+          soc.close
+        end
+      end
+    end
+  end
+end

data/lib/dtr/shared/configuration.rb

@@ -0,0 +1,76 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'singleton'
+require 'rinda/rinda'
+require 'rinda/tuplespace'
+
+module DTR
+
+  def configuration
+    Configuration.instance
+  end
+
+  module_function :configuration
+
+  class Configuration
+    include Singleton
+    include Service::Rinda
+
+    attr_accessor :broadcast_list, :rinda_server_port, :master_yell_interval, :follower_listen_sleep_timeout
+
+    def initialize
+      store = EnvStore.new
+      @broadcast_list = ['localhost'].concat(store[:broadcast_list] || []).uniq
+      @rinda_server_port = store[:port] || 3344
+      @master_yell_interval = store[:master_yell_interval] || 10
+      @follower_listen_sleep_timeout =  store[:follower_listen_sleep_timeout] || 15
+    end
+
+    def save
+      store = EnvStore.new
+      store[:broadcast_list] = @broadcast_list
+      store[:port] = @rinda_server_port
+      store[:master_yell_interval] = @master_yell_interval
+      store[:follower_listen_sleep_timeout] = @follower_listen_sleep_timeout
+    end
+
+    def with_rinda_server(&block)
+      start_service
+      DTR.info '-- Booting DTR Rinda server...'
+      loop do
+        begin
+          Rinda::RingServer.new Rinda::TupleSpace.new, @rinda_server_port
+          break
+        rescue Errno::EADDRINUSE
+          @rinda_server_port += 1
+        end
+      end
+      DTR.info "-- DTR Rinda server started on port #{@rinda_server_port}"
+      block.call
+    ensure
+      stop_service rescue nil
+    end
+
+    def lookup_ring_any
+      @ring ||= __lookup_ring_any__
+    end
+
+    private
+    def __lookup_ring_any__
+      DTR.info {"broadcast list: #{@broadcast_list.inspect} on port #{@rinda_server_port}"}
+      ::Rinda::TupleSpaceProxy.new(Rinda::RingFinger.new(@broadcast_list, @rinda_server_port).lookup_ring_any)
+    end
+  end
+end

data/lib/dtr/shared/message_decorator.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module MessageDecorator
+    def decorate_message(msg, source=nil)
+      source ? "#{source} from #{Socket.gethostname}: #{msg}" : "From #{Socket.gethostname}: #{msg}"
+    end
+  end
+  class RemoteError < StandardError
+    include MessageDecorator
+    def initialize(e)
+      super(decorate_message(e.message, e.class.name))
+      set_backtrace(e.backtrace)
+    end
+  end
+end

data/lib/dtr/shared/ruby_ext.rb

@@ -0,0 +1,153 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Null
+  class << self
+    def instance(overrides = {})
+      self.new.define_overrides(overrides)
+    end  
+  end
+
+  #override Object#id for removing the warning
+  def id
+    nil
+  end
+
+  def method_missing(sym, *args, &block)
+    nil
+  end
+
+  def define_overrides(overrides)
+    overrides.each_pair do |key, value|
+      (class << self; self; end;).send(:define_method, key, lambda { value })
+    end
+    self
+  end
+end
+
+# from activesupport-2.1.1
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end
+
+class NilClass
+  def blank?
+    true
+  end
+end
+
+class FalseClass
+  def blank?
+    true
+  end
+end
+
+class TrueClass
+  def blank?
+    false
+  end
+end
+
+class Array
+  alias_method :blank?, :empty?
+end
+
+class Hash
+  alias_method :blank?, :empty?
+end
+
+class String
+  def blank?
+    self !~ /\S/
+  end
+end
+
+class Numeric
+  def blank?
+    false
+  end
+end
+
+class Module
+  # Encapsulates the common pattern of:
+  #
+  #   alias_method :foo_without_feature, :foo
+  #   alias_method :foo, :foo_with_feature
+  #
+  # With this, you simply do:
+  #
+  #   alias_method_chain :foo, :feature
+  #
+  # And both aliases are set up for you.
+  #
+  # Query and bang methods (foo?, foo!) keep the same punctuation:
+  #
+  #   alias_method_chain :foo?, :feature
+  #
+  # is equivalent to
+  #
+  #   alias_method :foo_without_feature?, :foo?
+  #   alias_method :foo?, :foo_with_feature?
+  #
+  # so you can safely chain foo, foo?, and foo! with the same feature.
+  def alias_method_chain(target, feature)
+    # Strip out punctuation on predicates or bang methods since
+    # e.g. target?_without_feature is not a valid method name.
+    aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
+    yield(aliased_target, punctuation) if block_given?
+    
+    with_method, without_method = "#{aliased_target}_with_#{feature}#{punctuation}", "#{aliased_target}_without_#{feature}#{punctuation}"
+    
+    alias_method without_method, target
+    alias_method target, with_method
+    
+    case
+      when public_method_defined?(without_method)
+        public target
+      when protected_method_defined?(without_method)
+        protected target
+      when private_method_defined?(without_method)
+        private target
+    end
+  end
+
+  # Allows you to make aliases for attributes, which includes 
+  # getter, setter, and query methods.
+  #
+  # Example:
+  #
+  #   class Content < ActiveRecord::Base
+  #     # has a title attribute
+  #   end
+  #
+  #   class Email < Content
+  #     alias_attribute :subject, :title
+  #   end
+  #
+  #   e = Email.find(1)
+  #   e.title    # => "Superstars"
+  #   e.subject  # => "Superstars"
+  #   e.subject? # => true
+  #   e.subject = "Megastars"
+  #   e.title    # => "Megastars"
+  def alias_attribute(new_name, old_name)
+    module_eval <<-STR, __FILE__, __LINE__+1
+      def #{new_name}; self.#{old_name}; end
+      def #{new_name}?; self.#{old_name}?; end
+      def #{new_name}=(v); self.#{old_name} = v; end
+    STR
+  end
+end

data/lib/dtr/shared/service/agent.rb

@@ -0,0 +1,33 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module Agent
+      include Rinda
+      def new_agent_monitor
+        lookup_ring.notify(nil, [:agent, nil])
+      end
+
+      def provide_agent_info(setup_env_cmd, runners)
+        agent = %{
+- agent(host at #{Socket.gethostname}):
+    default setup environment command: '#{setup_env_cmd}'
+    runners: #{runners.inspect}
+}
+        lookup_ring.write [:agent, agent]
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/file.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module File
+      include Rinda
+      def lookup_file
+        lookup(:read, [:file, nil])[1]
+      end
+
+      def provide_file(file)
+        lookup_ring.write [:file, file]
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/rinda.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'drb'
+require 'rinda/ring'
+
+module DTR
+  module Service
+    module Rinda
+
+      def start_service
+        DTR.info "-- Start drb service..."
+        DRb.start_service
+      end
+
+      def stop_service
+        DRb.stop_service
+      end
+
+      def lookup(method, stuff, timeout=nil)
+        lookup_ring.send(method, stuff, timeout)
+      end
+
+      def lookup_ring
+        DTR.configuration.lookup_ring_any
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/runner.rb

@@ -0,0 +1,33 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rinda/ring'
+
+module DTR
+  module Service
+    module Runner
+      include Rinda
+
+      def provide_runner(runner)
+        renewer = ::Rinda::SimpleRenewer.new
+        tuple = [:name, 'DTR::Runner'.to_sym, runner, "DTR remote runner #{Process.pid}-#{runner.name}"]
+        lookup_ring.write(tuple, renewer)
+      end
+
+      def lookup_runner
+        lookup(:take, [:name, 'DTR::Runner'.to_sym, nil, nil])[2]
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/working_env.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module WorkingEnv
+      include Rinda
+      def lookup_working_env
+        lookup(:read, [:working_env, nil])[1]
+      end
+
+      def provide_working_env(env)
+        lookup_ring.write [:working_env, env]
+      end
+    end
+  end
+end

data/lib/dtr/shared/service.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared/service/rinda'
+require 'dtr/shared/service/file'
+require 'dtr/shared/service/runner'
+require 'dtr/shared/service/working_env'
+require 'dtr/shared/service/agent'

data/lib/dtr/shared/sync_codebase/codebase.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncCodebase
+    class Codebase
+      include DRbUndumped
+      include Package
+
+      CHUNK_SIZE = 1024*1024
+
+      def write(remote_io)
+        File.open(File.join(package_dir, package_file), "rb") do |f|
+          while chunk = f.read(CHUNK_SIZE) and chunk.length > 0
+            remote_io.write(chunk)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dtr/shared/sync_codebase/master_ext.rb

@@ -0,0 +1,52 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test/unit/testresult'
+
+module DTR
+  module SyncCodebase
+    module MasterExt
+      include Service::File
+
+      def self.included(base)
+        base.alias_method_chain :with_dtr_master, :sync_codebase
+      end
+
+      def with_dtr_master_with_sync_codebase(&block)
+        with_dtr_master_without_sync_codebase do
+          unless Cmd.execute('rake dtr_repackage --trace')
+            $stderr.puts %{
+Execute dtr_repackage rake task failed, see log for details.
+For running DTR test task, you must define a DTR::PackageTask task in your rakefile for DTR need package and synchronize your codebase within grid.
+Example:
+  require 'dtr/raketasks'
+  DTR::PackageTask.new do |p|
+    p.package_files.include("**/*")
+    p.package_files.exclude("tmp")
+    p.package_files.exclude("log")
+  end
+}
+            return Test::Unit::TestResult.new
+          end
+          begin
+            provide_file Codebase.new
+            block.call
+          ensure
+            Cmd.execute('rake dtr_clobber_package --trace')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dtr/shared/sync_codebase/package.rb

@@ -0,0 +1,39 @@
+# Copyright (c) 2007-2008 Li Xiao
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module SyncCodebase
+    module Package
+      def package_dir
+        'dtr_pkg'
+      end
+
+      def package_name
+        'codebase-dump'
+      end
+
+      def package_dir_path
+        "#{package_dir}/#{package_name}"
+      end
+
+      def package_file
+        "#{package_name}.tar.bz2"
+      end
+
+      def package_copy_file
+        "copy_#{package_file}"
+      end
+    end
+  end
+end