right_agent 1.0.1 → 2.0.7

data/lib/right_agent/history.rb

@@ -77,17 +77,17 @@ module RightScale
     #   :graceful_exits(Integer|nil):: Number of graceful terminations, if any
     #   :crashes(Integer|nil):: Number of crashes, if any
     #   :last_crash_time(Integer|nil):: Time in seconds in Unix-epoch when last crash occurred, if any
+    #   :crashed_last(Boolean):: Whether crashed last time it was started
     def analyze_service
       now = Time.now.to_i
       if @last_analysis && @last_event == @last_update
-        if @last_analysis[:uptime] > 0
-          delta = now - @last_analysis_time
-          @last_analysis[:uptime] += delta
-          @last_analysis[:total_uptime] += delta
-        end
+        delta = now - @last_analysis_time
+        @last_analysis[:uptime] += delta
+        @last_analysis[:total_uptime] += delta
       else
         last_run = last_crash = @last_event = {:time => 0, :pid => 0, :event => nil}
         restarts = graceful_exits = crashes = accumulated_uptime = 0
+        crashed_last = false
         load.each do |event|
           event = SerializationHelper.symbolize_keys(event)
           case event[:event]
@@ -98,19 +98,23 @@ module RightScale
             when "start"
               crashes += 1
               last_crash = event
+              crashed_last = true
             when "run"
               crashes += 1
               last_crash = event
+              crashed_last = true
               # Accumulating uptime here although this will wrongly include recovery time
               accumulated_uptime += (event[:time] - @last_event[:time])
             end
           when "run"
             last_run = event
           when "stop"
+            crashed_last = false
             if @last_event[:event] == "run" && @last_event[:pid] == event[:pid]
               accumulated_uptime += (event[:time] - @last_event[:time])
             end
           when "graceful exit"
+            crashed_last = false
             graceful_exits += 1
           else
             next
@@ -129,6 +133,7 @@ module RightScale
         if crashes > 0
           @last_analysis[:crashes] = crashes
           @last_analysis[:last_crash_time] = last_crash[:time]
+          @last_analysis[:crashed_last] = crashed_last
         end
       end
       @last_analysis_time = now

data/lib/right_agent/log.rb

@@ -133,7 +133,7 @@ module RightScale
     def method_missing(m, *args)
       init unless @initialized
       @logger.level = level_from_sym(level) if @level_frozen
-      @logger.__send__(m, *args)
+      @logger.send(m, *args)
     end
 
     # Determine whether this object, or its method_missing proxy, responds
@@ -218,9 +218,9 @@ module RightScale
     # lvl(Constant):: One of Logger::DEBUG ... Logger::FATAL
     #
     # === Raise
-    # (RightScale::Exceptions::Argument):: if level symbol is invalid
+    # (ArgumentError):: if level symbol is invalid
     def level_from_sym(sym)
-      raise Exceptions::Argument, "Invalid log level symbol :#{sym}" unless LEVELS_MAP.include?(sym)
+      raise ArgumentError, "Invalid log level symbol :#{sym}" unless LEVELS_MAP.include?(sym)
       lvl = LEVELS_MAP[sym]
     end
 
@@ -233,10 +233,10 @@ module RightScale
     # sym(Symbol):: One of :debug, :info, :warn, :error or :fatal
     #
     # === Raise
-    # (RightScale::Exceptions::Argument):: if level is invalid
+    # (ArgumentError):: if level is invalid
     def level_to_sym(lvl)
       @@inverted_levels_map ||= LEVELS_MAP.invert
-      raise Exceptions::Argument, "Invalid log level: #{lvl}" unless @@inverted_levels_map.include?(lvl)
+      raise ArgumentError, "Invalid log level: #{lvl}" unless @@inverted_levels_map.include?(lvl)
       sym = @@inverted_levels_map[lvl]
     end
 

data/lib/right_agent/minimal.rb

@@ -39,6 +39,7 @@ end
 # requires are oriented toward that. any additional use cases may require a
 # rethink of minimal loading.
 require ::File.normalize_path('agent_config', RIGHT_AGENT_BASE_DIR)
+require ::File.normalize_path('protocol_version_mixin', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('command', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('log', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('pid_file', RIGHT_AGENT_BASE_DIR)

data/lib/right_agent/multiplexer.rb

@@ -83,7 +83,7 @@ module RightScale
     # === Return
     # res(Object):: Result of first target in list
     def method_missing(m, *args)
-      res = @targets.inject([]) { |res, t| res << t.__send__(m, *args) }
+      res = @targets.inject([]) { |res, t| res << t.send(m, *args) }
       res[0]
     end
 

data/lib/right_agent/offline_handler.rb

@@ -0,0 +1,270 @@
+#
+# Copyright (c) 2009-2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Handler for queueing of requests when offline relative to RightNet
+  # and then sending the requests when successfully reconnect
+  class OfflineHandler
+
+    # Maximum seconds to wait before starting flushing offline queue when disabling offline mode
+    MAX_QUEUE_FLUSH_DELAY = 2 * 60
+
+    # Maximum number of offline queued requests before triggering restart vote
+    MAX_QUEUED_REQUESTS = 100
+
+    # Number of seconds that should be spent in offline mode before triggering a restart vote
+    RESTART_VOTE_DELAY = 15 * 60
+
+    # (Symbol) Current queue state with possible values:
+    #   Value          Description                Action               Next state
+    #   :created       Queue created              init                 :initializing
+    #   :initializing  Agent still initializing   start                :running
+    #   :running       Queue has been started     disable when offline :flushing
+    #   :flushing      Sending queued requests    enable               :running
+    #   :terminating   Agent terminating
+    attr_reader :state
+
+    # (Symbol) Current offline handling mode with possible values:
+    #   Value          Description
+    #   :initializing  Agent still initializing
+    #   :online        Agent connected
+    #   :offline       Agent disconnected
+    attr_reader :mode
+
+    # (Array) Offline queue
+    attr_accessor :queue
+
+    # Create offline queueing handler
+    #
+    # === Parameters
+    # restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
+    #   by offline queue exceeding MAX_QUEUED_REQUESTS
+    # offline_stats(RightSupport::Stats::Activity):: Offline queue tracking statistics
+    def initialize(restart_callback, offline_stats)
+      @restart_vote = restart_callback
+      @restart_vote_timer = nil
+      @restart_vote_count = 0
+      @offline_stats = offline_stats
+      @state = :created
+      @mode = :initializing
+      @queue = []
+    end
+
+    # Initialize the offline queue
+    # All requests sent prior to running this initialization are queued
+    # and then are sent once this initialization has run
+    # All requests following this call and prior to calling start
+    # are prepended to the request queue
+    #
+    # === Return
+    # true:: Always return true
+    def init
+      @state = :initializing if @state == :created
+      true
+    end
+
+    # Switch to online mode and send all buffered messages
+    #
+    # === Return
+    # true:: Always return true
+    def start
+      if @state == :initializing
+        if @mode == :offline
+          @state = :running
+        else
+          @state = :flushing
+          flush
+        end
+        @mode = :online if @mode == :initializing
+      end
+      true
+    end
+
+    # Is agent currently offline?
+    #
+    # === Return
+    # (Boolean):: true if agent offline, otherwise false
+    def offline?
+      @mode == :offline || @state == :created
+    end
+
+    # In request queueing mode?
+    #
+    # === Return
+    # (Boolean):: true if should queue request, otherwise false
+    def queueing?
+      offline? && @state != :flushing
+    end
+
+    # Switch to offline mode
+    # In this mode requests are queued in memory rather than being sent
+    # Idempotent
+    #
+    # === Return
+    # true:: Always return true
+    def enable
+      if offline?
+        if @state == :flushing
+          # If we were in offline mode then switched back to online but are still in the
+          # process of flushing the in-memory queue and are now switching to offline mode
+          # again then stop the flushing
+          @state = :running
+        end
+      else
+        Log.info("[offline] Disconnect from RightNet detected, entering offline mode")
+        Log.info("[offline] Messages will be queued in memory until RightNet connection is re-established")
+        @offline_stats.update
+        @queue ||= []  # Ensure queue is valid without losing any messages when going offline
+        @mode = :offline
+        start_timer
+      end
+      true
+    end
+
+    # Switch back to sending requests after in-memory queue gets flushed
+    # Idempotent
+    #
+    # === Return
+    # true:: Always return true
+    def disable
+      if offline? && @state != :created
+        Log.info("[offline] Connection to RightNet re-established")
+        @offline_stats.finish
+        cancel_timer
+        @state = :flushing
+        # Wait a bit to avoid flooding RightNet
+        EM.add_timer(rand(MAX_QUEUE_FLUSH_DELAY)) { flush }
+      end
+      true
+    end
+
+    # Queue given request in memory
+    #
+    # === Parameters
+    # request(Hash):: Request to be stored
+    #
+    # === Return
+    # true:: Always return true
+    def queue_request(kind, type, payload, target, callback)
+      request = {:kind => kind, :type => type, :payload => payload, :target => target, :callback => callback}
+      Log.info("[offline] Queuing request: #{request.inspect}")
+      vote_to_restart if (@restart_vote_count += 1) >= MAX_QUEUED_REQUESTS
+      if @state == :initializing
+        # We are in the initialization callback, requests should be put at the head of the queue
+        @queue.unshift(request)
+      else
+        @queue << request
+      end
+      true
+    end
+
+    # Prepare for agent termination
+    #
+    # === Return
+    # true:: Always return true
+    def terminate
+      @state = :terminating
+      cancel_timer
+      true
+    end
+
+    protected
+
+    # Send any requests that were queued while in offline mode
+    # Do this asynchronously to allow for agents to respond to requests
+    # Once all in-memory requests have been flushed, switch off offline mode
+    #
+    # === Parameters
+    # again(Boolean):: Whether being called in a loop
+    #
+    # === Return
+    # true:: Always return true
+    def flush(again = false)
+      if @state == :flushing
+        Log.info("[offline] Starting to flush request queue of size #{@queue.size}") unless again || @mode == :initializing
+        if @queue.any?
+          r = @queue.shift
+          if r[:callback]
+            Sender.instance.send(r[:kind], r[:type], r[:payload], r[:target]) { |result| r[:callback].call(result) }
+          else
+            Sender.instance.send(r[:kind], r[:type], r[:payload], r[:target])
+          end
+        end
+        if @queue.empty?
+          Log.info("[offline] Request queue flushed, resuming normal operations") unless @mode == :initializing
+          @mode = :online
+          @state = :running
+        else
+          EM.next_tick { flush(true) }
+        end
+      end
+      true
+    end
+
+    # Vote for restart and reset trigger
+    #
+    # === Parameters
+    # timer_trigger(Boolean):: true if vote was triggered by timer, false if it
+    #   was triggered by number of messages in in-memory queue
+    #
+    # === Return
+    # true:: Always return true
+    def vote_to_restart(timer_trigger = false)
+      if @restart_vote
+        @restart_vote.call
+        if timer_trigger
+          start_timer
+        else
+          @restart_vote_count = 0
+        end
+      end
+      true
+    end
+
+    # Start restart vote timer
+    #
+    # === Return
+    # true:: Always return true
+    def start_timer
+      if @restart_vote && @state != :terminating
+        @restart_vote_timer ||= EM::Timer.new(RESTART_VOTE_DELAY) { vote_to_restart(timer_trigger = true) }
+      end
+      true
+    end
+
+    # Cancel restart vote timer
+    #
+    # === Return
+    # true:: Always return true
+    def cancel_timer
+      if @restart_vote_timer
+        @restart_vote_timer.cancel
+        @restart_vote_timer = nil
+        @restart_vote_count = 0
+      end
+      true
+    end
+
+  end # OfflineHandler
+
+end # RightScale

data/lib/right_agent/packets.rb

@@ -40,7 +40,7 @@ end
 
 module RightScale
 
-  # Base class for all packets flowing through the mappers
+  # Base class for all packets flowing through the RightNet routers
   # Knows how to dump itself to MessagePack or JSON
   class Packet
 
@@ -156,7 +156,7 @@ module RightScale
     # === Return
     # log_msg(String):: Log representation
     def to_s(filter = nil, version = nil)
-      v = __send__(version) if version
+      v = send(version) if version
       v = (v && v != DEFAULT_VERSION) ? " v#{v}" : ""
       log_msg = "[#{name}#{v}]"
       duration = if @duration && (filter.nil? || filter.include?(:duration))
@@ -295,8 +295,8 @@ module RightScale
     # opts(Hash):: Optional settings:
     #   :from(String):: Sender identity
     #   :scope(Hash):: Define behavior that should be used to resolve tag based routing
-    #   :token(String):: Generated request id that a mapper uses to identify replies
-    #   :reply_to(String):: Identity of the node that actor replies to, usually a mapper itself
+    #   :token(String):: Generated request id that a router uses to identify replies
+    #   :reply_to(String):: Identity of the node that actor replies to, usually a router itself
     #   :selector(Symbol):: Selector used to route the request: :any or :all, defaults to :any,
     #     :all deprecated for version 13 and above
     #   :target(String|Array):: Target recipient(s)
@@ -426,12 +426,12 @@ module RightScale
     # opts(Hash):: Optional settings:
     #   :from(String):: Sender identity
     #   :scope(Hash):: Define behavior that should be used to resolve tag based routing
-    #   :token(String):: Generated request id that a mapper uses to identify replies
+    #   :token(String):: Generated request id that a router uses to identify replies
     #   :selector(Symbol):: Selector used to route the request: :any or :all, defaults to :any
     #   :target(String|Array):: Target recipient(s)
     #   :persistent(Boolean):: Indicates if this request should be saved to persistent storage
     #     by the AMQP broker
-    #   :confirm(Boolean):: Whether require confirmation response from mapper containing targets
+    #   :confirm(Boolean):: Whether require confirmation response from router containing targets
     #     to which request was published but not necessarily delivered
     #   :expires_at(Integer|nil):: Time in seconds in Unix-epoch when this request expires and
     #      is to be ignored by the receiver; value 0 means never expire; defaults to 0
@@ -532,7 +532,7 @@ module RightScale
     # Create packet
     #
     # === Parameters
-    # token(String):: Generated request id that a mapper uses to identify replies
+    # token(String):: Generated request id that a router uses to identify replies
     # to(String):: Identity of the node to which result should be delivered
     # results(Any):: Arbitrary data that is transferred from actor, a result of actor's work
     # from(String):: Sender identity

data/lib/right_agent/payload_formatter.rb

@@ -39,7 +39,7 @@ module RightScale
       parts = type.split('/')
       meth = "#{parts[1]}_#{parts[2]}".to_sym
       res = nil
-      res = @formatter.__send__(meth, payload) if @formatter.respond_to?(meth)
+      res = @formatter.send(meth, payload) if @formatter.respond_to?(meth)
       res
     end
 

data/lib/right_agent/pending_requests.rb

@@ -0,0 +1,128 @@
+#
+# Copyright (c) 2009-2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Request that is waiting for a response
+  class PendingRequest
+
+    # (Symbol) Kind of request: :send_push or :send_request
+    attr_reader :kind
+
+    # (Time) Time when request message was received
+    attr_reader :receive_time
+
+    # (Proc) Block to be activated when response is received
+    attr_reader :response_handler
+
+    # (String) Token for parent request in a retry situation
+    attr_accessor :retry_parent_token
+
+    # (String) Non-delivery reason if any
+    attr_accessor :non_delivery
+
+    def initialize(kind, receive_time, response_handler)
+      @kind = kind
+      @receive_time = receive_time
+      @response_handler = response_handler
+      @retry_parent_token = nil
+      @non_delivery = nil
+    end
+
+  end # PendingRequest
+
+  # Cache for requests that are waiting for a response
+  # Automatically deletes push requests when get too old
+  # Retains non-push requests until explicitly deleted
+  class PendingRequests < Hash
+
+    # Maximum number of seconds to retain send pushes in cache
+    MAX_PUSH_AGE = 2 * 60
+
+    # Minimum number of seconds between push cleanups
+    MIN_CLEANUP_INTERVAL = 15
+
+    # Create cache
+    def initialize
+      @last_cleanup = Time.now
+      super
+    end
+
+    # Store pending request
+    #
+    # === Parameters
+    # token(String):: Generated message identifier
+    # pending_request(PendingRequest):: Pending request
+    #
+    # === Return
+    # (PendingRequest):: Stored request
+    def []=(token, pending_request)
+      now = Time.now
+      if (now - @last_cleanup) > MIN_CLEANUP_INTERVAL
+        self.reject! { |t, r| r.kind == :send_push && (now - r.receive_time) > MAX_PUSH_AGE }
+        @last_cleanup = now
+      end
+      super
+    end
+
+    # Select cache entries of the given kind
+    #
+    # === Parameters
+    # kind(Symbol):: Kind of request to be included: :send_push or :send_request
+    #
+    # === Return
+    # (Hash):: Requests of specified kind
+    def kind(kind)
+      self.reject { |t, r| r.kind != kind}
+    end
+
+    # Get age of youngest pending request
+    #
+    # === Return
+    # age(Integer):: Age of youngest request
+    def youngest_age
+      now = Time.now
+      age = nil
+      self.each_value do |r|
+        seconds = (now - r.receive_time).to_i
+        age = seconds if age.nil? || seconds < age
+      end
+      age
+    end
+
+    # Get age of oldest pending request
+    #
+    # === Return
+    # age(Integer):: Age of oldest request
+    def oldest_age
+      now = Time.now
+      age = nil
+      self.each_value do |r|
+        seconds = (now - r.receive_time).to_i
+        age = seconds if age.nil? || seconds > age
+      end
+      age
+    end
+
+  end # PendingRequests
+
+end # RightScale