right_agent 0.5.1 → 0.5.10

data/lib/right_agent.rb

@@ -20,31 +20,22 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require 'rubygems'
+# reuse minimal loader to ensure right_agent can safely be required after minimal.
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'minimal'))
+
 require 'amqp'
-require 'mq'
 require 'json'
 require 'yaml'
 require 'openssl'
 
-# Cannot use File.normalize_path here because not defined until after this include
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'platform'))
-
-unless defined?(RIGHT_AGENT_BASE_DIR)
-  RIGHT_AGENT_BASE_DIR = File.normalize_path(File.join(File.dirname(__FILE__), 'right_agent'))
-end
-
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'monkey_patches'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_config'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'payload_formatter'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'packets'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'enrollment_result'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'console'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'daemonize'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'pid_file'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'exceptions'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'multiplexer'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'log'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'tracer'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'audit_formatter'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize'))
@@ -54,7 +45,6 @@ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'subprocess'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'stats_helper'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'broker_client'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'ha_broker_client'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'command'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_identity'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_tags_manager'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor'))

data/lib/right_agent/actors/agent_manager.rb

@@ -30,7 +30,8 @@ class AgentManager
 
   on_exception { |meth, deliverable, e| RightScale::ExceptionMailer.deliver_notification(meth, deliverable, e) }
 
-  expose :ping, :stats, :set_log_level, :execute, :connect, :disconnect, :terminate
+  expose :ping, :stats, :profile, :set_log_level, :execute, :connect, :disconnect, :connect_failed,
+         :tune_heartbeat, :terminate
 
   # Valid log levels
   LEVELS = [:debug, :info, :warn, :error, :fatal]
@@ -68,6 +69,48 @@ class AgentManager
     @agent.stats(RightScale::SerializationHelper.symbolize_keys(options))
   end
 
+  # Profile memory use
+  #
+  # === Parameters
+  # options(Hash):: Request options
+  #   :start(Boolean):: Whether to start profiling
+  #   :stats(Boolean):: Whether to display profile statistics to stdout
+  #   :reset(Boolean):: Whether to reset profile statistics when after displaying them
+  #   :stop(Boolean):: Whether to stop profiling
+  #
+  # === Return
+  # (OperationResult):: Empty success result or error result with message
+  def profile(options)
+    require 'memprof'
+
+    options = RightScale::SerializationHelper.symbolize_keys(options)
+    if options[:start]
+      RightScale::Log.info("[profile] Start")
+      $stderr.puts "[profile] Start at #{Time.now}"
+      Memprof.start
+      @profiling = true
+    end
+
+    if options[:stats]
+      return error_result("Profiling has not yet been started") unless @profiling
+      RightScale::Log.info("[profile] GC start")
+      $stderr.puts "[profile] GC at #{Time.now}"
+      GC.start
+      RightScale::Log.info("[profile] Display stats to stderr")
+      $stderr.puts "[profile] Stats at #{Time.now}#{options[:reset] ? ' with reset' : ''}"
+      options[:reset] ? Memprof.stats! : Memprof.stats
+    end
+
+    if options[:stop]
+      return error_result("Profiling has not yet been started") unless @profiling
+      RightScale::Log.info("[profile] Stop")
+      $stderr.puts "[profile] Stop at #{Time.now}"
+      Memprof.stop
+      @profiling = false
+    end
+    success_result
+  end
+
   # Change log level of agent
   #
   # === Parameter
@@ -123,7 +166,7 @@ class AgentManager
         res = error_result(error)
       end
     rescue Exception => e
-      res = error_result("Failed to connect to broker: #{e.message}")
+      res = error_result("Failed to connect to broker", e)
     end
     res
   end
@@ -146,7 +189,7 @@ class AgentManager
         res = error_result(error)
       end
     rescue Exception => e
-      res = error_result("Failed to disconnect from broker: #{e.message}")
+      res = error_result("Failed to disconnect from broker", e)
     end
     res
   end
@@ -167,7 +210,38 @@ class AgentManager
         res = error_result(error)
       end
     rescue Exception => e
-      res = error_result("Failed to notify agent that brokers #{options[:brokers]} are unusable: #{e.message}")
+      res = error_result("Failed to notify agent that brokers #{options[:brokers]} are unusable", e)
+    end
+    res
+  end
+
+  # Tune connection heartbeat frequency for all brokers
+  # Any response to this request is not likely to get through if :immediate is specified
+  # because the broker connections will be in flux
+  # Use of :immediate should be avoided when this is a fanned out request to avoid
+  # overloading the brokers
+  #
+  # === Parameters
+  # options(Hash):: Tune options:
+  #   :heartbeat(Integer):: New AMQP connection heartbeat setting, nil or 0 means disable
+  #   :immediate(Boolean):: Whether to tune heartbeat immediately rather than defer until next
+  #     status check
+  #
+  # === Return
+  # res(RightScale::OperationResult):: Success unless exception is raised
+  def tune_heartbeat(options)
+    options = RightScale::SerializationHelper.symbolize_keys(options)
+    res = success_result
+    begin
+      if options[:immediate]
+        if error = @agent.tune_heartbeat(options[:heartbeat])
+          res = error_result(error)
+        end
+      else
+        @agent.defer_task { @agent.tune_heartbeat(options[:heartbeat]) }
+      end
+    rescue Exception => e
+      res = error_result("Failed to tune heartbeat", e)
     end
     res
   end

data/lib/right_agent/agent.rb

@@ -73,6 +73,7 @@ module RightScale
       :check_interval     => 5 * 60,
       :grace_timeout      => 30,
       :prefetch           => 1,
+      :heartbeat          => 60
     }
 
     # Initializes a new agent and establishes an AMQP connection.
@@ -97,12 +98,14 @@ module RightScale
     #   :retry_timeout(Numeric):: Maximum number of seconds to retry request before give up
     #   :time_to_live(Integer):: Number of seconds before a request expires and is to be ignored
     #     by the receiver, 0 means never expire, defaults to 0
-    #   :connect_timeout:: Number of seconds to wait for a broker connection to be established
+    #   :connect_timeout(Integer):: Number of seconds to wait for a broker connection to be established
     #   :reconnect_interval(Integer):: Number of seconds between broker reconnect attempts
     #   :ping_interval(Integer):: Minimum number of seconds since last message receipt to ping the mapper
     #     to check connectivity, defaults to 0 meaning do not ping
-    #   :check_interval:: Number of seconds between publishing stats and checking for broker connections
+    #   :check_interval(Integer):: Number of seconds between publishing stats and checking for broker connections
     #     that failed during agent launch and then attempting to reconnect via the mapper
+    #   :heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
+    #     connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
     #   :grace_timeout(Integer):: Maximum number of seconds to wait after last request received before
     #     terminating regardless of whether there are still unfinished requests
     #   :dup_check(Boolean):: Whether to check for and reject duplicate requests, e.g., due to retries
@@ -153,6 +156,7 @@ module RightScale
       @tags << opts[:tag] if opts[:tag]
       @tags.flatten!
       @options.freeze
+      @deferred_tasks = []
       @terminating = false
       @last_stat_reset_time = @service_start_time = Time.now
       reset_agent_stats
@@ -239,6 +243,57 @@ module RightScale
       @registry.register(actor, prefix)
     end
 
+    # Tune connection heartbeat frequency for all brokers
+    # Causes a reconnect to each broker
+    #
+    # === Parameters
+    # heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
+    #   connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
+    #
+    # === Return
+    # res(String|nil):: Error message if failed, otherwise nil
+    def tune_heartbeat(heartbeat)
+      res = nil
+      begin
+        @broker.heartbeat = heartbeat
+        update_configuration(:heartbeat => heartbeat)
+        ids = []
+        all = @broker.all
+        all.each do |id|
+          begin
+            host, port, index, priority, island_id = @broker.identity_parts(id)
+            @broker.connect(host, port, index, priority, island = nil, force = true) do |id|
+              @broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
+                begin
+                  if status == :connected
+                    setup_queues([id])
+                    tuned = (heartbeat && heartbeat != 0) ? "Tuned heartbeat to #{heartbeat} seconds" : "Disabled heartbeat"
+                    Log.info("[setup] #{tuned} for broker #{id}")
+                  else
+                    Log.error("Failed to reconnect to broker #{id} to tune heartbeat, status #{status.inspect}")
+                  end
+                rescue Exception => e
+                  Log.error("Failed to setup queues for broker #{id} when tuning heartbeat", e, :trace)
+                  @exceptions.track("tune heartbeat", e)
+                end
+              end
+            end
+            ids << id
+          rescue Exception => e
+            res = Log.format("Failed to reconnect to broker #{id} to tune heartbeat", e)
+            Log.error("Failed to reconnect to broker #{id} to tune heartbeat", e, :trace)
+            @exceptions.track("tune heartbeat", e)
+          end
+        end
+        res = "Failed to tune heartbeat for brokers #{(all - ids).inspect}" unless (all - ids).empty?
+      rescue Exception => e
+        res = Log.format("Failed tuning broker connection heartbeat", e)
+        Log.error("Failed tuning broker connection heartbeat", e, :trace)
+        @exceptions.track("tune heartbeat", e)
+      end
+      res
+    end
+
     # Connect to an additional broker or reconnect it if connection has failed
     # Subscribe to identity queue on this broker
     # Update config file if this is a new broker
@@ -384,6 +439,17 @@ module RightScale
       true
     end
 
+    # Defer task until next status check
+    #
+    # === Block
+    # Required block to be activated on next status check
+    #
+    # === Return
+    # true:: Always return true
+    def defer_task(&task)
+      @deferred_tasks << task
+    end
+
     # Gracefully terminate execution by allowing unfinished tasks to complete
     # Immediately terminate if called a second time
     #
@@ -462,6 +528,7 @@ module RightScale
       result = OperationResult.success("name"            => @agent_name,
                                        "identity"        => @identity,
                                        "hostname"        => Socket.gethostname,
+                                       "memory"          => Platform.process.resident_set_size,
                                        "version"         => AgentConfig.protocol_version,
                                        "brokers"         => @broker.stats(reset),
                                        "agent stats"     => agent_stats(reset),
@@ -674,12 +741,22 @@ module RightScale
       true
     end
 
-    # Check status of agent by gathering current operation statistics and publishing them and
-    # by completing any queue setup that can be completed now based on broker status
+    # Check status of agent by gathering current operation statistics and publishing them,
+    # executing any deferred tasks, and finishing any queue setup
     #
     # === Return
     # true:: Always return true
     def check_status
+      @deferred_tasks.reject! do |t|
+        begin
+          t.call
+        rescue Exception => e
+          Log.error("Failed to perform deferred task", e)
+          @exceptions.track("check status", e)
+        end
+        true
+      end
+
       begin
         finish_setup
       rescue Exception => e

data/lib/right_agent/agent_config.rb

@@ -63,7 +63,7 @@ module RightScale
   module AgentConfig
 
     # Current agent protocol version
-    PROTOCOL_VERSION = 17
+    PROTOCOL_VERSION = 18
 
     # Current agent protocol version
     #
@@ -344,6 +344,22 @@ module RightScale
       options || {}
     end
 
+    # Agents that are currently running
+    #
+    # === Parameters
+    # (Regexp):: Pattern that agent name must match to be included
+    #
+    # === Return
+    # (Array):: Name of running agents
+    def self.running_agents(pattern = //)
+      AgentConfig.cfg_agents.select do |agent_name|
+        agent_name =~ pattern &&
+        (pid_file = AgentConfig.pid_file(agent_name)) &&
+        (pid = pid_file.read_pid[:pid]) &&
+        (Process.getpgid(pid) rescue -1) != -1
+      end.sort
+    end
+
     protected
 
     # Convert value to array if not an array, unless nil

data/lib/right_agent/agent_tags_manager.rb

@@ -40,7 +40,7 @@ module RightScale
     # === Return
     # true:: Always return true
     def tags
-      do_query do |result|
+      do_query(nil, @agent.identity) do |result|
         if result.kind_of?(Hash)
           yield(result.size == 1 ? result.values.first['tags'] : [])
         else
@@ -178,7 +178,7 @@ module RightScale
     # true:: Always return true
     def do_query(tags = nil, agent_ids = nil, raw = false)
       agent_check
-      payload = {:agent_ids => [@agent.identity]}
+      payload = {:agent_identity => @agent.identity}
       payload[:tags] = ensure_flat_array_value(tags) unless tags.nil? || tags.empty?
       payload[:agent_ids] = ensure_flat_array_value(agent_ids) unless agent_ids.nil? || agent_ids.empty?
       request = RightScale::IdempotentRequest.new("/mapper/query_tags", payload)

data/lib/right_agent/broker_client.rb

@@ -37,8 +37,8 @@ module RightScale
       :failed        # Failed to connect due to internal failure or AMQP failure to connect
     ]
 
-    # (MQ) AMQP client
-    attr_reader :mq
+    # (AMQP::Channel) Channel of AMQP connection used by this client
+    attr_reader :channel
 
     # (String) Broker identity
     attr_reader :identity
@@ -99,6 +99,8 @@ module RightScale
     #   :vhost(String):: Virtual host path name
     #   :insist(Boolean):: Whether to suppress redirection of connection
     #   :reconnect_interval(Integer):: Number of seconds between reconnect attempts
+    #   :heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
+    #     connection alive, e.g., when AMQP broker is behind a firewall
     #   :prefetch(Integer):: Maximum number of messages the AMQP broker is to prefetch for the agent
     #     before it receives an ack. Value 1 ensures that only last unacknowledged gets redelivered
     #     if the agent crashes. Value 0 means unlimited prefetch.
@@ -217,13 +219,13 @@ module RightScale
 
       begin
         Log.info("[setup] Subscribing queue #{queue[:name]}#{to_exchange} on broker #{@alias}")
-        q = @mq.queue(queue[:name], queue_options)
+        q = @channel.queue(queue[:name], queue_options)
         @queues << q
         if exchange
-          x = @mq.__send__(exchange[:type], exchange[:name], exchange_options)
+          x = @channel.__send__(exchange[:type], exchange[:name], exchange_options)
           binding = q.bind(x, options[:key] ? {:key => options[:key]} : {})
           if exchange2 = options[:exchange2]
-            q.bind(@mq.__send__(exchange2[:type], exchange2[:name], exchange2[:options] || {}))
+            q.bind(@channel.__send__(exchange2[:type], exchange2[:name], exchange2[:options] || {}))
           end
           q = binding
         end
@@ -319,8 +321,8 @@ module RightScale
       return false unless usable?
       begin
         Log.info("[setup] Declaring #{name} #{type.to_s} on broker #{@alias}")
-        delete_from_cache(:queue, name)
-        @mq.__send__(type, name, options)
+        delete_amqp_resources(:queue, name)
+        @channel.__send__(type, name, options)
         true
       rescue Exception => e
         Log.error("Failed declaring #{type.to_s} #{name} on broker #{@alias}", e, :trace)
@@ -361,8 +363,8 @@ module RightScale
         end
         Log.debug("... publish options #{options.inspect}, exchange #{exchange[:name]}, " +
                   "type #{exchange[:type]}, options #{exchange[:options].inspect}")
-        delete_from_cache(exchange[:type], exchange[:name]) if exchange_options[:declare]
-        @mq.__send__(exchange[:type], exchange[:name], exchange_options).publish(message, options)
+        delete_amqp_resources(exchange[:type], exchange[:name]) if exchange_options[:declare]
+        @channel.__send__(exchange[:type], exchange[:name], exchange_options).publish(message, options)
         true
       rescue Exception => e
         Log.error("Failed publishing to exchange #{exchange.inspect} on broker #{@alias}", e, :trace)
@@ -390,7 +392,7 @@ module RightScale
     # === Return
     # true:: Always return true
     def return_message
-      @mq.return_message do |info, message|
+      @channel.return_message do |info, message|
         begin
           to = if info.exchange && !info.exchange.empty? then info.exchange else info.routing_key end
           reason = info.reply_text
@@ -418,14 +420,14 @@ module RightScale
         begin
           @queues.reject! do |q|
             if q.name == name
-              @mq.queue(name, options.merge(:no_declare => true)).delete
+              @channel.queue(name, options.merge(:no_declare => true)).delete
               deleted = true
             end
           end
           unless deleted
             # Allowing declare to happen since queue may not exist and do not want NOT_FOUND
             # failure to cause AMQP channel to close
-            @mq.queue(name, options).delete
+            @channel.queue(name, options).delete
             deleted = true
           end
         rescue Exception => e
@@ -436,6 +438,19 @@ module RightScale
       deleted
     end
 
+    # Delete resources from local AMQP cache
+    #
+    # === Parameters
+    # type(Symbol):: Type of AMQP object
+    # name(String):: Name of object
+    #
+    # === Return
+    # true:: Always return true
+    def delete_amqp_resources(type, name)
+      @channel.__send__(type == :queue ? :queues : :exchanges).delete(name)
+      true
+    end
+
     # Close broker connection
     #
     # === Parameters
@@ -520,7 +535,7 @@ module RightScale
     # Makes client callback with :connected or :disconnected status if boundary crossed
     #
     # === Parameters
-    # status(Symbol):: Status of connection (:connected, :ready, :disconnected, :stopping, :failed, :closed)
+    # status(Symbol):: Status of connection (:connected, :disconnected, :stopping, :failed, :closed)
     #
     # === Return
     # true:: Always return true
@@ -528,11 +543,6 @@ module RightScale
       # Do not let closed connection regress to failed
       return true if status == :failed && @status == :closed
 
-      # Wait until connection is ready (i.e. handshake with broker is completed) before
-      # changing our status to connected
-      return true if status == :connected
-      status = :connected if status == :ready
-
       before = @status
       @status = status
 
@@ -574,11 +584,12 @@ module RightScale
                                    :host               => address[:host],
                                    :port               => address[:port],
                                    :insist             => @options[:insist] || false,
+                                   :heartbeat          => @options[:heartbeat],
                                    :reconnect_delay    => lambda { rand(reconnect_interval) },
                                    :reconnect_interval => reconnect_interval)
-        @mq = MQ.new(@connection)
-        @mq.__send__(:connection).connection_status { |status| update_status(status) }
-        @mq.prefetch(@options[:prefetch]) if @options[:prefetch]
+        @channel = AMQP::Channel.new(@connection)
+        @channel.__send__(:connection).connection_status { |status| update_status(status) }
+        @channel.prefetch(@options[:prefetch]) if @options[:prefetch]
       rescue Exception => e
         @status = :failed
         @failures.update
@@ -653,19 +664,6 @@ module RightScale
       true
     end
 
-    # Delete object from local AMQP cache in case it is no longer consistent with broker
-    #
-    # === Parameters
-    # type(Symbol):: Type of AMQP object
-    # name(String):: Name of object
-    #
-    # === Return
-    # true:: Always return true
-    def delete_from_cache(type, name)
-      @mq.__send__(type == :queue ? :queues : :exchanges).delete(name)
-      true
-    end
-
     # Execute packet receive callback, make it a separate method to ease instrumentation
     #
     # === Parameters