right_agent 2.1.5 → 2.2.0

data/lib/right_agent/clients/blocking_client.rb

@@ -112,7 +112,7 @@ module RightScale
       result
     end
 
-    # Make long-polling requests until receive data or timeout
+    # Make long-polling requests until receive data, hit error, or timeout
     #
     # @param [Hash] connection to server from previous request with keys :host, :path,
     #   and :expires_at, with the :expires_at being adjusted on return
@@ -131,6 +131,16 @@ module RightScale
       [result, code, body, headers]
     end
 
+    # Close all persistent connections
+    #
+    # @param [String] reason for closing
+    #
+    # @return [TrueClass] always true
+    def close(reason)
+      @connections = {}
+      true
+    end
+
     protected
 
     # Make HTTP request once

data/lib/right_agent/clients/non_blocking_client.rb

@@ -123,17 +123,15 @@ module RightScale
       # Make request an then yield fiber until it completes
       fiber = Fiber.current
       connection = EM::HttpRequest.new(uri.to_s, connect_options)
+      # Store connection now so that close will get called if terminating or reconnecting
+      c = @connections[path] = {:host => host, :connection => connection, :expires_at => Time.now} if request_options[:keepalive]
       http = connection.send(verb, request_options)
-      http.errback { fiber.resume(http.error.to_s == "Errno::ETIMEDOUT" ? 504 : 500,
-                                  (http.error && http.error.to_s) || "HTTP connection failure for #{verb.to_s.upcase}") }
+      http.errback { @connections.delete(path); fiber.resume(*handle_error(verb, http.error)) }
       http.callback { fiber.resume(http.response_header.status, http.response, http.response_header) }
       response_code, response_body, response_headers = Fiber.yield
       response_headers = beautify_headers(response_headers) if response_headers
       result = BalancedHttpClient.response(response_code, response_body, response_headers, request_options[:head][:accept])
-      if request_options[:keepalive]
-        expires_at = Time.now + BalancedHttpClient::CONNECTION_REUSE_TIMEOUT
-        @connections[path] = {:host => host, :connection => connection, :expires_at => expires_at}
-      end
+      c[:expires_at] = Time.now + BalancedHttpClient::CONNECTION_REUSE_TIMEOUT if request_options[:keepalive]
       [result, response_code, response_body, response_headers]
     end
 
@@ -159,9 +157,21 @@ module RightScale
       [result, code, body, headers]
     end
 
+    # Close all persistent connections
+    #
+    # @param [String] reason for closing
+    #
+    # @return [TrueClass] always true
+    def close(reason)
+      @connections.each_value { |c| c[:connection].close(reason) }
+      @connections = {}
+      true
+    end
+
     protected
 
-    # Repeatedly make long-polling request until receive data or timeout
+    # Repeatedly make long-polling request until receive data, hit error, or timeout
+    # Treat "terminating" and "reconnecting" errors as an empty poll result
     #
     # @param [Symbol] verb for HTTP REST request
     # @param [EM:HttpRequest] connection to server from previous request
@@ -173,8 +183,7 @@ module RightScale
     # @raise [HttpException] HTTP failure with associated status code
     def poll_again(fiber, connection, request_options, stop_at)
       http = connection.send(:get, request_options)
-      http.errback { fiber.resume(http.error.to_s == "Errno::ETIMEDOUT" ? 504 : 500,
-                                  (http.error && http.error.to_s) || "HTTP connection failure for POLL") }
+      http.errback { fiber.resume(*handle_error("POLL", http.error)) }
       http.callback do
         code, body, headers = http.response_header.status, http.response, http.response_header
         if code == 200 && (body.nil? || body == "null") && Time.now < stop_at
@@ -186,6 +195,20 @@ module RightScale
       true
     end
 
+    # Handle error from request
+    #
+    # @param [Symbol] verb for HTTP REST request
+    # @param [Object] error result from HTTP connection
+    #
+    # @return [Array] status code and error message string
+    def handle_error(verb, error)
+      case error.to_s
+      when "terminating", "reconnecting" then [200, nil]
+      when "Errno::ETIMEDOUT" then [408, "Request timeout"]
+      else [500, (error && error.to_s) || "HTTP connection failure for #{verb.to_s.upcase}"]
+      end
+    end
+
     # Beautify response header keys so that in same form as RestClient
     #
     # @param [Hash] headers from response

data/lib/right_agent/clients/right_http_client.rb

@@ -93,8 +93,11 @@ module RightScale
     #       Packet::GLOBAL, ones with no shard id
     #   [Symbol] :selector for picking from qualified targets: :any or :all;
     #     defaults to :any
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [NilClass] always nil since there is no expected response to the request
     #
@@ -105,10 +108,10 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def push(type, payload = nil, target = nil, token = nil)
+    def push(type, payload = nil, target = nil, options = {})
       raise RuntimeError, "#{self.class.name}#init was not called" unless @auth
       client = (@api && @api.support?(type)) ? @api : @router
-      client.push(type, payload, target, token)
+      client.push(type, payload, target, options)
     end
 
     # Route a request to a single target with a response expected
@@ -127,8 +130,11 @@ module RightScale
     #   [Array] :tags that must all be associated with a target for it to be selected
     #   [Hash] :scope for restricting routing which may contain:
     #     [Integer] :account id that agents must be associated with to be included
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [Result, NilClass] response from request
     #
@@ -139,10 +145,10 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def request(type, payload = nil, target = nil, token = nil)
+    def request(type, payload = nil, target = nil, options = {})
       raise RuntimeError, "#{self.class.name}#init was not called" unless @auth
       client = (@api && @api.support?(type)) ? @api : @router
-      client.request(type, payload, target, token)
+      client.request(type, payload, target, options)
     end
 
     # Route event

data/lib/right_agent/clients/router_client.rb

@@ -54,7 +54,7 @@ module RightScale
     RECONNECT_INTERVAL = 2
 
     # Maximum interval between attempts to reconnect or long-poll when router is not responding
-    MAX_RECONNECT_INTERVAL = 60
+    MAX_RECONNECT_INTERVAL = 30
 
     # Interval between checks for lost WebSocket connection
     CHECK_INTERVAL = 5
@@ -116,8 +116,11 @@ module RightScale
     #       Packet::GLOBAL, ones with no shard id
     #   [Symbol] :selector for picking from qualified targets: :any or :all;
     #     defaults to :any
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [NilClass] always nil since there is no expected response to the request
     #
@@ -127,12 +130,12 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def push(type, payload, target, token = nil)
+    def push(type, payload, target, options = {})
       params = {
         :type => type,
         :payload => payload,
         :target => target }
-      make_request(:post, "/push", params, type.split("/")[2], token)
+      make_request(:post, "/push", params, type.split("/")[2], options)
     end
 
     # Route a request to a single target with a response expected
@@ -152,8 +155,11 @@ module RightScale
     #   [Array] :tags that must all be associated with a target for it to be selected
     #   [Hash] :scope for restricting routing which may contain:
     #     [Integer] :account id that agents must be associated with to be included
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [Result, NilClass] response from request
     #
@@ -163,12 +169,12 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def request(type, payload, target, token = nil)
+    def request(type, payload, target, options = {})
       params = {
         :type => type,
         :payload => payload,
         :target => target }
-      make_request(:post, "/request", params, type.split("/")[2], token)
+      make_request(:post, "/request", params, type.split("/")[2], options)
     end
 
     # Route event
@@ -198,7 +204,7 @@ module RightScale
         Log.info("Sending EVENT <#{event[:uuid]}> #{event[:type]}#{path}#{to}")
         @websocket.send(JSON.dump(params))
       else
-        make_request(:post, "/notify", params, "notify", event[:uuid], :filter_params => ["event"])
+        make_request(:post, "/notify", params, "notify", :request_uuid => event[:uuid], :filter_params => ["event"])
       end
       true
     end
@@ -344,11 +350,32 @@ module RightScale
         @listen_interval = CHECK_INTERVAL
       end
 
-      # Loop using next_tick or timer
+      listen_loop_wait(Time.now, @listen_interval, routing_keys, &handler)
+    end
+
+    # Wait specified interval before next listen loop
+    # Continue waiting if interval changes while waiting
+    #
+    # @param [Time] started_at time when first started waiting
+    # @param [Numeric] interval to wait
+    # @param [Array, NilClass] routing_keys for event sources of interest with nil meaning all
+    #
+    # @yield [event] required block called each time event received
+    # @yieldparam [Hash] event received
+    #
+    # @return [TrueClass] always true
+    def listen_loop_wait(started_at, interval, routing_keys, &handler)
       if @listen_interval == 0
         EM_S.next_tick { listen_loop(routing_keys, &handler) }
       else
-        @listen_timer = EM_S::Timer.new(@listen_interval) { listen_loop(routing_keys, &handler) }
+        @listen_timer = EM_S::Timer.new(interval) do
+          remaining = @listen_interval - (Time.now - started_at)
+          if remaining > 0
+            listen_loop_wait(started_at, remaining, routing_keys, &handler)
+          else
+            listen_loop(routing_keys, &handler)
+          end
+        end
       end
       true
     end
@@ -555,7 +582,7 @@ module RightScale
         :poll_timeout => @options[:listen_timeout] }
 
       event_uuids = []
-      events = make_request(:poll, "/listen", params, "listen", nil, options)
+      events = make_request(:poll, "/listen", params, "listen", options)
       if events
         events.each do |event|
           event = SerializationHelper.symbolize_keys(event)
@@ -610,7 +637,7 @@ module RightScale
     #
     # @return [Boolean] true if router not responding, otherwise false
     def router_not_responding?
-      @close_code == PROTOCOL_ERROR_CLOSE && @close_reason =~ /502|503/
+      @close_code == PROTOCOL_ERROR_CLOSE && @close_reason =~ /408|502|503/
     end
 
   end # RouterClient

data/lib/right_agent/command/command_client.rb

@@ -82,6 +82,7 @@ module RightScale
         @pending += 1
         command = options.dup
         command[:verbose] = verbose
+        command[:timeout] = timeout
         command[:cookie] = @cookie
         EM.next_tick { EM.connect('127.0.0.1', @socket_port, ConnectionHandler, command, self, response_handler) }
         EM.add_timer(timeout) { EM.stop; raise 'Timed out waiting for agent reply' } if manage_em

data/lib/right_agent/http_exceptions.rb

@@ -69,7 +69,12 @@ module RightScale
 
     # Convert RestClient exception
     def self.convert(e)
-      e2 = create(e.http_code, e.http_body, RightScale::Response.new((e.response && e.response.headers) || {}))
+      e2 = if e.is_a?(RestClient::RequestTimeout)
+        # Special case RequestTimeout because http_code and http_body is typically nil given no actual response
+        create(408)
+      else
+        create(e.http_code, e.http_body, RightScale::Response.new((e.response && e.response.headers) || {}))
+      end
       e2.message = e.message
       e2
     end

data/lib/right_agent/monkey_patches/ruby_patch/windows_patch/process_patch.rb

@@ -44,12 +44,12 @@ begin
         @@win32_kill.call(sig, *pids)
       end
 
-      # implements getpgid() for Windws
+      # implements getpgid() for Windows
       def self.getpgid(pid)
         # FIX: we currently only use this to check if the process is running.
         # it is possible to get the parent process id for a process in Windows if
         # we actually need this info.
-        return Process.kill(0, pid).contains?(pid) ? 0 : -1
+        return Process.kill(0, pid).include?(pid) ? 0 : -1
       rescue
         raise Errno::ESRCH
       end

data/lib/right_agent/offline_handler.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2013 RightScale Inc
+# Copyright (c) 2009-2014 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -27,7 +27,7 @@ module RightScale
   class OfflineHandler
 
     # Maximum seconds to wait before starting flushing offline queue when disabling offline mode
-    MAX_QUEUE_FLUSH_DELAY = 2 * 60
+    MAX_QUEUE_FLUSH_DELAY = 60
 
     # Maximum number of offline queued requests before triggering restart vote
     MAX_QUEUED_REQUESTS = 100
@@ -161,12 +161,23 @@ module RightScale
     # Queue given request in memory
     #
     # === Parameters
-    # request(Hash):: Request to be stored
+    # kind(Symbol):: Kind of request: :send_push or :send_request
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(Hash|NilClass):: Target for request
+    # token(String):: Token uniquely identifying request
+    # expires_at(Integer):: Time in seconds in Unix-epoch when this request expires and
+    #   is to be ignored by the receiver; value 0 means never expire
+    #
+    # === Block
+    # Optional block used to process response asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR
     #
     # === Return
     # true:: Always return true
-    def queue_request(kind, type, payload, target, callback)
-      request = {:kind => kind, :type => type, :payload => payload, :target => target, :callback => callback}
+    def queue_request(kind, type, payload, target, token, expires_at, &callback)
+      request = {:kind => kind, :type => type, :payload => payload, :target => target,
+                 :token => token, :expires_at => expires_at, :callback => callback}
       Log.info("[offline] Queuing request: #{request.inspect}")
       vote_to_restart if (@restart_vote_count += 1) >= MAX_QUEUED_REQUESTS
       if @state == :initializing
@@ -190,7 +201,7 @@ module RightScale
 
     protected
 
-    # Send any requests that were queued while in offline mode
+    # Send any requests that were queued while in offline mode and have not yet timed out
     # Do this asynchronously to allow for agents to respond to requests
     # Once all in-memory requests have been flushed, switch off offline mode
     #
@@ -204,10 +215,12 @@ module RightScale
         Log.info("[offline] Starting to flush request queue of size #{@queue.size}") unless again || @mode == :initializing
         if @queue.any?
           r = @queue.shift
-          if r[:callback]
-            Sender.instance.send(r[:kind], r[:type], r[:payload], r[:target]) { |result| r[:callback].call(result) }
+          options = {:token => r[:token]}
+          if r[:expires_at] != 0 && (options[:time_to_live] = r[:expires_at] - Time.now.to_i) <= 0
+            Log.info("[offline] Dropping queued request <#{r[:token]}> because it expired " +
+                     "#{(-options[:time_to_live]).round} sec ago")
           else
-            Sender.instance.send(r[:kind], r[:type], r[:payload], r[:target])
+            Sender.instance.send(r[:kind], r[:type], r[:payload], r[:target], options, &r[:callback])
           end
         end
         if @queue.empty?

data/lib/right_agent/retryable_request.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2014 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -74,13 +74,13 @@ module RightScale
     # options(Hash):: Request options
     #   :targets(Array):: Target agent identities from which to randomly choose one
     #   :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
-    #   :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
+    #   :retry_delay(Numeric):: Number of seconds delay before initial retry with -1 meaning no delay,
     #     defaults to DEFAULT_RETRY_DELAY
-    #   :retry_delay_count(Fixnum):: Minimum number of retries at initial :retry_delay value before
+    #   :retry_delay_count(Numeric):: Minimum number of retries at initial :retry_delay value before
     #     increasing delay exponentially and decreasing this count exponentially, defaults to
     #     DEFAULT_RETRY_DELAY_COUNT
-    #   :max_retry_delay(Fixnum):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
-    #   :timeout(Fixnum):: Number of seconds with no response before error callback gets called, with
+    #   :max_retry_delay(Numeric):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
+    #   :timeout(Numeric):: Number of seconds with no response before error callback gets called, with
     #     -1 meaning never, defaults to DEFAULT_TIMEOUT
     #
     # === Raises
@@ -90,6 +90,7 @@ module RightScale
       raise ArgumentError.new("payload is required") unless (@payload = payload)
       @retry_on_error = options[:retry_on_error]
       @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @expires_at = Time.now.to_i + @timeout if @timeout > 0
       @retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
       @retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT
       @max_retry_delay = options[:max_retry_delay] || DEFAULT_MAX_RETRY_DELAY
@@ -105,13 +106,18 @@ module RightScale
     # === Return
     # true:: Always return true
     def run
-      Sender.instance.send_request(@operation, @payload, retrieve_target(@targets)) { |r| handle_response(r) }
-      if @cancel_timer.nil? && @timeout > 0
-        @cancel_timer = EM::Timer.new(@timeout) do
-          msg = "Request #{@operation} timed out after #{@timeout} seconds"
-          Log.info(msg)
-          cancel(msg)
-        end
+      cancel = Proc.new do
+        msg = "Request #{@operation} timed out after #{@timeout} seconds"
+        Log.info(msg)
+        cancel(msg)
+      end
+
+      options = {}
+      if @expires_at.nil? || (options[:time_to_live] = @expires_at - Time.now.to_i) > 0
+        Sender.instance.send_request(@operation, @payload, retrieve_target(@targets), options) { |r| handle_response(r) }
+        @cancel_timer = EM::Timer.new(@timeout) { cancel.call } if @cancel_timer.nil? && @timeout > 0
+      else
+        cancel.call
       end
       true
     end

data/lib/right_agent/scripts/agent_controller.rb

@@ -404,9 +404,15 @@ module RightScale
         pgid = Process.getpgid(pid) rescue -1
         name = human_readable_name(agent_name, pid_file.identity)
         if pgid != -1
-          psdata = `ps up #{pid}`.split("\n").last.split
-          memory = (psdata[5].to_i / 1024)
-          puts "#{name} is alive, using #{memory}MB of memory"
+          message = "#{name} is alive"
+          unless RightScale::Platform.windows?
+            # Windows Platform code currently does not support retrieving memory usage
+            # information for another process, so only include it for linux
+            psdata = `ps up #{pid}`.split("\n").last.split
+            memory = (psdata[5].to_i / 1024)
+            message << ", using #{memory}MB of memory"
+          end
+          puts message
           res = true
         else
           puts "#{name} is not running but has a stale pid file at #{pid_file}"