pylon 0.2.7 → 0.2.8

data/lib/pylon.rb

@@ -16,5 +16,5 @@
 
 class Pylon
   PYLON_ROOT = File.dirname(File.expand_path(File.dirname(__FILE__)))
-  VERSION = "0.2.7"
+  VERSION = "0.2.8"
 end

data/lib/pylon/application.rb

@@ -119,6 +119,17 @@ class Pylon
     :description => "Group to set privilege to",
     :proc => nil
 
+    option :multicast,
+    :short => "-M",
+    :long => "--multicast",
+    :description => "Enable multicast support via encapuslated pragmatic general multicast",
+    :proc => lambda { |m| true }
+
+    option :multicast_interface,
+    :short => "-i INTERFACE",
+    :long => "--multicast-interface INTERFACE",
+    :description => "Interface to use to send multicast over"
+
     option :multicast_address,
     :short => "-a ADDRESS",
     :long => "--multicast-address ADDRESS",
@@ -135,16 +146,11 @@ class Pylon
     :description => "Enable multicast over loopback interfaces",
     :proc => lambda { |loop| true }
 
-    option :interface,
-    :short => "-i INTERFACE",
-    :long => "--interface INTERFACE",
-    :description => "Interface to use to send multicast over"
-
     option :tcp_address,
     :short => "-t TCPADDRESS",
     :long => "--tcp-address TCPADDRESS",
     :description => "Interface to use to bind request socket to"
-    
+
     option :tcp_port,
     :short => "-P TCPPORT",
     :long => "--tcp-port TCPPORT",
@@ -155,7 +161,6 @@ class Pylon
     :long => "--minimum-master-nodes NODES",
     :description => "How many nodes to wait for before starting master election",
     :proc => lambda { |nodes| nodes.to_i }
-
   end
 end
 

data/lib/pylon/command.rb

@@ -0,0 +1,133 @@
+# Author:: AJ Christensen (<aj@junglist.gen.nz>)
+# Copyright:: Copyright (c) 2011 AJ Christensen
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or#implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative "../pylon" # For the PYLON_ROOT const
+require_relative "log"
+require_relative "exceptions"
+require_relative "mixin/convert_to_class_name"
+
+class Pylon
+  class Command
+
+    extend Pylon::Mixin::ConvertToClassName
+
+    attr_accessor :options, :config
+
+    def self.options
+      @options ||= {}
+      @options
+    end
+
+    def self.options=(val)
+      raise(ArgumentError, "Options must recieve a hash") unless val.kind_of?(Hash)
+      @options = val
+    end
+
+    def initialize(*args)
+      @options = Hash.new
+      @config = Hash.new
+
+      klass_options = self.class.options
+      klass_options.keys.inject(@options) { |memo,key| memo[key] = klass_options[key].dup; memo }
+
+      @options.each do |config_key, config_opts|
+        config[config_key] = config_opts
+      end
+
+      super(*args)
+    end
+
+    def self.inherited(subclass)
+      unless subclass.unnamed?
+        commands[subclass.snake_case_name] = subclass
+      end
+    end
+
+    def self.commands
+      @@commands ||= {}
+    end
+
+    def self.unnamed?
+      name.nil? or name.empty?
+    end
+
+    def self.snake_case_name
+      convert_to_snake_case(name.split('::').last) unless unnamed?
+    end
+
+    def self.common_name
+      snake_case_name.split('_').join(' ')
+    end
+
+    def self.load_commands
+      command_files.each do |file|
+        Kernel.load file
+      end
+      true
+    end
+
+    def self.command_files
+      @@command_files ||= find_commands.values.flatten.uniq
+    end
+
+    def self.find_commands
+      files = Dir[File.expand_path('../command/*.rb', __FILE__)]
+      command_files = {}
+      files.each do |command_file|
+        rel_path = command_file[/#{Pylon::PYLON_ROOT}#{Regexp.escape(File::SEPARATOR)}(.*)\.rb/,1]
+        command_files[rel_path] = command_file
+      end
+      command_files
+    end
+
+    def self.list_commands
+      load_commands
+      commands.each do |command|
+        Log.info "command loaded: #{command}"
+      end
+    end
+
+    def self.run(command, options={})
+      Log.warn "command: options is not a hash" unless options.is_a? Hash
+      load_commands
+      command_class = command_class_from(command)
+      command_class.options = options.merge!(command_class.options) if options.respond_to? :merge! # just in case 
+      instance = command_class.new(command)
+      instance.run
+    end
+
+    def self.command_class_from(args)
+      args = [args].flatten
+      command_words = args.select {|arg| arg =~ /^(([[:alnum:]])[[:alnum:]\_\-]+)$/ }
+      command_class = nil
+      while ( !command_class ) && ( !command_words.empty? )
+        snake_case_class_name = command_words.join("_")
+        unless command_class = commands[snake_case_class_name]
+          command_words.pop
+        end
+      end
+      command_class ||= commands[args.first.gsub('-', '_')]
+      command_class || command_not_found!(args)
+    end
+
+    def self.command_not_found!(args)
+      Log.debug "command not found: #{args.inspect}"
+      raise Pylon::Exceptions::Command::NotFound, args
+    end
+
+  end # Command
+end # Pylon
+

data/lib/pylon/command/new_leader.rb

@@ -0,0 +1,26 @@
+# Author:: AJ Christensen (<aj@junglist.gen.nz>)
+# Copyright:: Copyright (c) 2011 AJ Christensen
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or#implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+class Pylon
+  class Command
+    class NewLeader < Command
+      def run
+        # raise InvalidOptions unless options.has_key? :new_leader
+        [ "ok", "new_leader" ].to_json
+      end
+    end
+  end
+end

data/lib/pylon/command/ping.rb

@@ -0,0 +1,26 @@
+# Author:: AJ Christensen (<aj@junglist.gen.nz>)
+# Copyright:: Copyright (c) 2011 AJ Christensen
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or#implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+class Pylon
+  class Command
+    class Ping < Command
+      def run
+        # for calculating if timestamp is within time bounds
+        [ "ok", :timestamp => Time.now.to_i ]
+      end
+    end
+  end
+end

data/lib/pylon/command/status.rb

@@ -0,0 +1,26 @@
+# Author:: AJ Christensen (<aj@junglist.gen.nz>)
+# Copyright:: Copyright (c) 2011 AJ Christensen
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or#implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+class Pylon
+  class Command
+    class Status < Command
+      def run
+        raise InvalidOptions unless options.has_key? :node
+        options[:node]
+      end
+    end
+  end
+end

data/lib/pylon/command_handler.rb

@@ -0,0 +1,30 @@
+# Author:: AJ Christensen (<aj@junglist.gen.nz>)
+# Copyright:: Copyright (c) 2011 AJ Christensen
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or#implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative "log"
+require_relative "command"
+
+class Pylon
+  class CommandHandler
+    def initialize
+      Log.debug "command_handler: initialized"
+    end
+
+    def handle_command
+      true
+    end
+  end
+end

data/lib/pylon/config.rb

@@ -22,7 +22,7 @@ class Pylon
     extend Mixlib::Config
 
     config_file "~/.pylon.rb"
-    log_level :info
+    log_level :debug
     log_location STDOUT
     daemonize false
     user nil
@@ -30,13 +30,13 @@ class Pylon
     umask 0022
 
     # Options for the multicast server
-    multicast true
+    multicast false
     multicast_address "225.4.5.6"
     multicast_port "13336"
     multicast_ttl 3
     multicast_listen_address nil
     multicast_loopback false
-    interface_address "127.0.0.1"
+    multicast_interface "eth0"
 
     # TCP settings
     tcp_address "*"
@@ -47,7 +47,7 @@ class Pylon
     # cluster settings
     maximum_weight 1000
     cluster_name "pylon"
-    Seed_tcp_endpoints []
+    seed_unicast_endpoints []
     master nil
     minimum_master_nodes 1
     sleep_after_announce 5
@@ -57,5 +57,5 @@ class Pylon
     fd_timeout 30
     fd_retries 3
 
-  end
-end
+  end # Config
+end # Pylon

data/lib/pylon/elector.rb

@@ -24,7 +24,7 @@ require_relative "node"
 class Pylon
   class Elector
 
-    attr_accessor :cluster_name, :context, :multicast_endpoint, :node, :nodes, :multicast_announcer_thread, :multicast_listener_thread, :tcp_listener_thread, :master
+    attr_accessor :cluster_name, :context, :multicast_endpoint, :node, :nodes, :multicast_announcer_thread, :multicast_listener_thread, :tcp_listener_thread
 
     def initialize
       @cluster_name = Pylon::Config[:cluster_name]
@@ -32,22 +32,21 @@ class Pylon
 
       @node = Pylon::Node.new
       @nodes = Array(@node)
-      @master = Pylon::Config[:master]
+      node.master = Pylon::Config[:master]
 
+      Pylon::Log.info "#{node} initialized, starting elector"
       Pylon::Log.info "elector[#{cluster_name}] initialized, starting pub/sub sockets on #{multicast_endpoint} and tcp listener socket on #{node.unicast_endpoint}"
 
       Thread.abort_on_exception = true
 
-      scheduler = Thread.new do
-        @unicast_announcer_thread = node.unicast_announcer
+      @unicast_announcer_thread = node.unicast_announcer
+      if Pylon::Config[:multicast]
         @multicast_announcer_thread = node.multicast_announcer
         @multicast_listener_thread = multicast_listener
-
-        @multicast_listener_thread.join
-        @unicast_announcer_thread.join
         @multicast_announcer_thread.join
+        @multicast_listener_thread.join
       end
-      scheduler.join
+      @unicast_announcer_thread.join
 
       # join listeners
       # @unicast_announcer_thread.join
@@ -84,42 +83,39 @@ class Pylon
     end
 
     def multicast_endpoint
-      @multicast_endpoint ||= "epgm://#{Pylon::Config[:interface]};#{Pylon::Config[:multicast_address]}:#{Pylon::Config[:multicast_port]}"
+      @multicast_endpoint ||= "epgm://#{Pylon::Config[:multicast_interface]};#{Pylon::Config[:multicast_address]}:#{Pylon::Config[:multicast_port]}"
+    end
+    
+    def failure_detectors
+      nodes.reject{|n| n == node}.map do |node|
+        Thread.new do
+          Log.info "failure_detector: starting failure detection against #{node}"
+          loop do
+            node.ping
+          end
+        end
+      end
     end
 
-    def add_node node
-      nodes << node unless nodes.include? node
-      # fire up a new failure detector
+    def refresh_failure_detectors
       failure_detectors.each do |thread|
         thread.join
       end
     end
+    
+    def add_node node
+      nodes << node unless nodes.include? node
+      allocate_master
+      # refresh_failure_detectors
+    end
 
-    def ping_node node
-      begin
-        Pylon::Config[:fd_retries].times do |attempt|
-          begin
-            Timeout::timeout(Pylon::Config[:fd_timeout]) do
-              pong, timestamp = node.send "ping", :attempt => attempt
-              if (timestamp - Time.now.to_i) >= 600
-                Log.warn "failure_detector: received bad timestamp from #{node}, sending 'exit' message"
-                node.send "exit", {"message" => "bad timestamp received after #{Pylon::Config[:fd_retries]}"}
-                nodes.delete node
-              else
-                Log.debug "failure_detector: received good pong with timestamp: #{timestamp}"
-                Thread.pass
-              end
-            end
-          rescue Timeout::Error
-            Log.warn "failure_detector: #{node} timed out, removing"
-            nodes.delete node
-          end
-        end
-      end
+    def remove_node node
+      nodes.delete node if nodes.include? node
+      # refresh_failure_detectors
     end
 
     def assert_leadership
-      nodes.each do |node|
+      nodes.map do |node|
         Thread.new do
           status = node.send "status"
           Log.info "assert_leadership: status of #{node}: #{status}"
@@ -128,23 +124,9 @@ class Pylon
         end
       end.each do |thread|
         thread.join
-      end if master
-    end
-
-    def failure_detectors
-      nodes.reject{|n| n == node}.map do |node|
-        Thread.new do
-          Log.info "failure_detector: starting failure detection against #{node}"
-          loop do
-            assert_leadership
-            ping_node node
-            allocate_master
-          end
-          Thread.pass
-        end
-      end
+      end if node.master
     end
-
+    
     def multicast_listener
       Thread.new do
         Log.debug "multicast_listener: zeromq sub socket starting up on #{multicast_endpoint}"
@@ -155,9 +137,7 @@ class Pylon
         sub_socket.setsockopt ZMQ::MCAST_LOOP, Pylon::Config[:multicast_loopback]
         sub_socket.connect multicast_endpoint
         loop do
-          uuid = sub_socket.recv_string
-          Log.debug "multicast_listener: handling announce from #{uuid}"
-          handle_announce sub_socket.recv_string if sub_socket.more_parts?
+          handle_announce sub_socket.recv_string
         end
       end
     end
@@ -167,30 +147,32 @@ class Pylon
         Log.debug "allocate_master: node: #{node}"
       end
       if node.uuid == nodes.last.uuid
-        @master = true
+        node.master = true
         Log.info "allocate_master: master allocated; sending new_leader"
-        nodes.each do |node|
-          node.send "new_leader", node
+        nodes.each do |remote_node|
+          remote_node.send "new_leader", :new_leader => node
         end
       else
         Log.info "allocate_master: someone else is the master, getting ready for work"
-        @master = false
+        node.master = false
       end
     end
 
-    def handle_announce recv_string
+    def handle_announce recv_string = ""
+      return false if recv_string.empty?
+      
       Log.info "handle_announce: got string #{recv_string}"
       new_node = JSON.parse(recv_string)
       Log.info "handle_anounce: got announce from #{new_node}"
-      if master
+      if node.master
         Log.info "handle_announce: I am the master: updating #{new_node} of leadership status"
         if node.weight > new_node.weight
           Log.info "handle_announce: I'm bigger than you: sending new_leader to #{new_node}"
-          new_node.send "new_leader", node
+          new_node.send "new_leader", :new_leader => node
         else
           Log.info "handle_announce: new leader, sending change_leader to all nodes"
           nodes.each do |n|
-            n.send "change_leader", new_node
+            n.send "change_leader", :new_leader => node
           end
         end
       elsif nodes.length < Pylon::Config[:minimum_master_nodes]
@@ -204,10 +186,10 @@ class Pylon
       end
     end
 
-
     def connect_node node
       Log.debug "connect_node: request socket connecting to #{node}"
-      new_node = node.send "status"
+      # parse a fresh node out of the status
+      new_node = JSON.parse(node.send "status")
       Log.debug "connect_node: node: #{new_node}"
       if nodes.include? new_node
         Log.info "connect_node: skipping node #{new_node}, already in local list, sleeping for 60 secs"