dtr 0.0.4 → 1.0.0

data/lib/dtr/shared/configuration.rb

@@ -0,0 +1,104 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'singleton'
+require 'rinda/ring'
+require 'rinda/tuplespace'
+
+module DTR
+
+  def configuration
+    Configuration.instance
+  end
+
+  module_function :configuration
+
+  # DTR Configuration includes: 
+  #   For both master & agent process:
+  #     * broadcast_list: broadcast ip list for looking for dtr services(agent and DTR master rinda server).
+  #     * group: grouping agents for specific usage, master process should specify same group for catching agents to work
+  #   For master process:
+  #     * rinda_server_port: DTR would automatically detect unused port for master rinda server. The default port is 3344.
+  #     * master_heartbeat_interval: master process heartbeat for keeping agents working. The default is 10 sec.
+  #   For agent process:
+  #     * agent_listen_port: agent listen master process command port, the default port is 7788.
+  #     * follower_listen_heartbeat_timeout: agents would be going to sleep if listening master heartbeat timeout. The default is 15 sec.
+  #
+  # All configurations except rinda_server_port would be stored into pstore file .dtr_env_pstore in the DTR process launching directory.
+  #
+  class Configuration
+    include Singleton
+    include Service::Rinda
+
+    attr_accessor :broadcast_list, :rinda_server_port, :agent_listen_port, :master_heartbeat_interval, :follower_listen_heartbeat_timeout
+    attr_reader :group
+
+    def initialize
+      load
+    end
+
+    def load
+      store = EnvStore.new
+      # always have 'localhost' in broadcast_list, for our master process would start rinda server locally,
+      # and dtr should work well on local machine when the machine leaves dtr grid network environment.
+      @broadcast_list = ['localhost'].concat(store[:broadcast_list] || []).uniq
+      @rinda_server_port = 3344
+      @agent_listen_port = store[:agent_listen_port] || 7788
+      @master_heartbeat_interval = store[:master_heartbeat_interval] || 10
+      @follower_listen_heartbeat_timeout =  store[:follower_listen_heartbeat_timeout] || 15
+      @group = store[:group]
+    end
+
+    def save
+      store = EnvStore.new
+      store[:broadcast_list] = @broadcast_list
+      store[:agent_listen_port] = @agent_listen_port
+      store[:master_heartbeat_interval] = @master_heartbeat_interval
+      store[:follower_listen_heartbeat_timeout] = @follower_listen_heartbeat_timeout
+      store[:group] = @group
+    end
+
+    def group=(group)
+      @group = group.blank? ? nil : group.gsub(' ', '_')
+    end
+
+    def with_rinda_server(&block)
+      DTR.do_println("Booting DTR service")
+      start_service
+      DTR.info {'-- Booting DTR Rinda server'}
+      loop do
+        begin
+          Rinda::RingServer.new Rinda::TupleSpace.new, @rinda_server_port
+          break
+        rescue Errno::EADDRINUSE
+          @rinda_server_port += 1
+        end
+      end
+      DTR.info {"-- DTR Rinda server started on port #{@rinda_server_port}"}
+      block.call
+    ensure
+      stop_service rescue nil
+    end
+
+    def lookup_ring_any
+      @ring ||= __lookup_ring_any__
+    end
+
+    private
+    def __lookup_ring_any__
+      DTR.info {"broadcast list: #{@broadcast_list.inspect} on port #{@rinda_server_port}"}
+      ::Rinda::TupleSpaceProxy.new(Rinda::RingFinger.new(@broadcast_list, @rinda_server_port).lookup_ring_any)
+    end
+  end
+end

data/lib/dtr/shared/message_decorator.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module MessageDecorator
+    def decorate_message(msg, source=nil)
+      source ? "#{source} from #{Socket.gethostname}: #{msg}" : "From #{Socket.gethostname}: #{msg}"
+    end
+  end
+  class RemoteError < StandardError
+    include MessageDecorator
+    def initialize(e)
+      super(decorate_message(e.message, e.class.name))
+      set_backtrace(e.backtrace)
+    end
+  end
+end

data/lib/dtr/shared/ruby_ext.rb

@@ -0,0 +1,129 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# from activesupport-2.1.1
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end
+
+class NilClass
+  def blank?
+    true
+  end
+end
+
+class FalseClass
+  def blank?
+    true
+  end
+end
+
+class TrueClass
+  def blank?
+    false
+  end
+end
+
+class Array
+  alias_method :blank?, :empty?
+end
+
+class Hash
+  alias_method :blank?, :empty?
+end
+
+class String
+  def blank?
+    self !~ /\S/
+  end
+end
+
+class Numeric
+  def blank?
+    false
+  end
+end
+
+class Module
+  # Encapsulates the common pattern of:
+  #
+  #   alias_method :foo_without_feature, :foo
+  #   alias_method :foo, :foo_with_feature
+  #
+  # With this, you simply do:
+  #
+  #   alias_method_chain :foo, :feature
+  #
+  # And both aliases are set up for you.
+  #
+  # Query and bang methods (foo?, foo!) keep the same punctuation:
+  #
+  #   alias_method_chain :foo?, :feature
+  #
+  # is equivalent to
+  #
+  #   alias_method :foo_without_feature?, :foo?
+  #   alias_method :foo?, :foo_with_feature?
+  #
+  # so you can safely chain foo, foo?, and foo! with the same feature.
+  def alias_method_chain(target, feature)
+    # Strip out punctuation on predicates or bang methods since
+    # e.g. target?_without_feature is not a valid method name.
+    aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
+    yield(aliased_target, punctuation) if block_given?
+    
+    with_method, without_method = "#{aliased_target}_with_#{feature}#{punctuation}", "#{aliased_target}_without_#{feature}#{punctuation}"
+    
+    alias_method without_method, target
+    alias_method target, with_method
+    
+    case
+      when public_method_defined?(without_method)
+        public target
+      when protected_method_defined?(without_method)
+        protected target
+      when private_method_defined?(without_method)
+        private target
+    end
+  end
+
+  # Allows you to make aliases for attributes, which includes 
+  # getter, setter, and query methods.
+  #
+  # Example:
+  #
+  #   class Content < ActiveRecord::Base
+  #     # has a title attribute
+  #   end
+  #
+  #   class Email < Content
+  #     alias_attribute :subject, :title
+  #   end
+  #
+  #   e = Email.find(1)
+  #   e.title    # => "Superstars"
+  #   e.subject  # => "Superstars"
+  #   e.subject? # => true
+  #   e.subject = "Megastars"
+  #   e.title    # => "Megastars"
+  def alias_attribute(new_name, old_name)
+    module_eval <<-STR, __FILE__, __LINE__+1
+      def #{new_name}; self.#{old_name}; end
+      def #{new_name}?; self.#{old_name}?; end
+      def #{new_name}=(v); self.#{old_name} = v; end
+    STR
+  end
+end

data/lib/dtr/shared/service.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dtr/shared/service/rinda'
+require 'dtr/shared/service/file'
+require 'dtr/shared/service/runner'
+require 'dtr/shared/service/working_env'
+require 'dtr/shared/service/agent'

data/lib/dtr/shared/service/agent.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module Agent
+      include Rinda
+      def new_agent_monitor
+        lookup_ring.notify(nil, [:agent, nil])
+      end
+
+      def provide_agent_info(setup_env_cmd, runners)
+        agent = %{
+- agent(host at #{Socket.gethostname}):
+    default setup environment command: '#{setup_env_cmd}'
+    runners: #{runners.inspect}
+}
+        lookup_ring.write [:agent, agent]
+      end
+
+      def all_agents_info
+        lookup_ring.read_all([:agent, nil]).collect{|t|t[1]}
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/file.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module File
+      include Rinda
+      def lookup_file
+        lookup(:read, [:file, nil])[1]
+      end
+
+      def provide_file(file)
+        lookup_ring.write [:file, file]
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/rinda.rb

@@ -0,0 +1,48 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'drb'
+require 'rinda/ring'
+
+module DTR
+  module Service
+    module Rinda
+
+      def start_service
+        DTR.info {"-- Start drb service..."}
+        #todo need figure why agents couldn't be killed directly when start drb service by DRb.start_service
+        if ENV['DTR_ENV'] == 'test'
+          DRb.start_service("druby://#{Socket.gethostname}:0")
+        else
+          DRb.start_service
+        end
+      rescue
+        # for ruby 1.8.7 need specify uri
+        DRb.start_service("druby://#{Socket.gethostname}:0")
+      end
+
+      def stop_service
+        DRb.stop_service
+      end
+
+      def lookup(method, stuff, timeout=nil)
+        lookup_ring.send(method, stuff, timeout)
+      end
+
+      def lookup_ring
+        DTR.configuration.lookup_ring_any
+      end
+    end
+  end
+end

data/lib/dtr/shared/service/runner.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rinda/ring'
+
+module DTR
+  module Service
+    module Runner
+      include Rinda
+
+      def provide_runner(runner)
+        tuple = ['DTR::Runner'.to_sym, runner, "DTR remote runner #{Process.pid}-#{runner.name}"]
+        #expires after 1 sec for we don't need runner service anymore if there is no one is waiting for taking it
+        lookup_ring.write(tuple, 1)
+      end
+
+      def lookup_runner
+        lookup(:take, ['DTR::Runner'.to_sym, nil, nil])[1]
+      end
+
+    end
+  end
+end

data/lib/dtr/shared/service/working_env.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module DTR
+  module Service
+    module WorkingEnv
+      include Rinda
+      def lookup_working_env
+        lookup(:read, [:working_env, nil])[1]
+      end
+
+      def provide_working_env(env)
+        lookup_ring.write [:working_env, env]
+      end
+    end
+  end
+end