right_agent 0.6.3 → 0.6.6

data/lib/right_agent.rb

@@ -24,6 +24,7 @@
 require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'minimal'))
 
 require 'amqp'
+require 'mq'
 require 'json'
 require 'yaml'
 require 'openssl'

data/lib/right_agent/agent_tags_manager.rb

@@ -184,7 +184,7 @@ module RightScale
       request = RightScale::IdempotentRequest.new("/mapper/query_tags", payload)
       request.callback { |result| yield raw ? request.raw_response : result }
       request.errback do |message|
-        RightScale::RightLinkLog.error("Failed to query tags: #{message}")
+        Log.error("Failed to query tags: #{message}")
         yield((raw ? request.raw_response : nil) || message)
       end
       request.run

data/lib/right_agent/broker_client.rb

@@ -543,6 +543,11 @@ module RightScale
       # Do not let closed connection regress to failed
       return true if status == :failed && @status == :closed
 
+      # Wait until connection is ready (i.e. handshake with broker is completed) before
+      # changing our status to connected
+      return true if status == :connected
+      status = :connected if status == :ready
+
       before = @status
       @status = status
 
@@ -587,7 +592,7 @@ module RightScale
                                    :heartbeat          => @options[:heartbeat],
                                    :reconnect_delay    => lambda { rand(reconnect_interval) },
                                    :reconnect_interval => reconnect_interval)
-        @channel = AMQP::Channel.new(@connection)
+        @channel = MQ.new(@connection)
         @channel.__send__(:connection).connection_status { |status| update_status(status) }
         @channel.prefetch(@options[:prefetch]) if @options[:prefetch]
       rescue Exception => e

data/lib/right_agent/core_payload_types.rb

@@ -27,6 +27,7 @@ require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'cookbook'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'cookbook_repository'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'cookbook_sequence'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'cookbook_position'))
+require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'dev_repository'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'dev_repositories'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'executable_bundle'))
 require File.normalize_path(File.join(CORE_PAYLOAD_TYPES_BASE_DIR, 'event_categories'))

data/lib/right_agent/core_payload_types/dev_repositories.rb

@@ -21,6 +21,8 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+require File.join(File.dirname(__FILE__), 'dev_repository')
+
 module RightScale
   # Sequence of cookbooks to be checked out on the instance.
   class DevRepositories
@@ -76,7 +78,15 @@ module RightScale
     # result(Hash):: The entry added to the collection of repositories
     def add_repo(repo_sha, repo_detail, cookbook_positions)
       @repositories ||= {}
-      @repositories[repo_sha] = { :repo => repo_detail, :positions => cookbook_positions }
+      @repositories[repo_sha] = DevRepository.new(repo_detail[:repo_type],
+                                                  repo_detail[:url],
+                                                  repo_detail[:tag],
+                                                  repo_detail[:cookboooks_path],
+                                                  repo_detail[:ssh_key],
+                                                  repo_detail[:username],
+                                                  repo_detail[:password],
+                                                  repo_sha,
+                                                  cookbook_positions)
     end
 
     #

data/lib/right_agent/core_payload_types/dev_repository.rb

@@ -0,0 +1,76 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScale
+  # Sequence of cookbooks to be checked out on the instance.
+  class DevRepository
+    include Serializable
+
+    # (Symbol) Type of repository: one of :git, :svn, :download or :local
+    # * :git denotes a 'git' repository that should be retrieved via 'git clone'
+    # * :svn denotes a 'svn' repository that should be retrieved via 'svn checkout'
+    # * :download denotes a tar ball that should be retrieved via HTTP GET (HTTPS if uri starts with https://)
+    # * :local denotes cookbook that is already local and doesn't need to be retrieved
+    attr_accessor :repo_type
+    # (String) URL to repository (e.g. git://github.com/opscode/chef-repo.git)
+    attr_accessor :url
+    # (String) git commit or svn branch that should be used to retrieve repository
+    # Optional, use 'master' for git and 'trunk' for svn if tag is nil.
+    # Not used for raw repositories.
+    attr_accessor :tag
+    # (Array) Path to cookbooks inside repostory
+    # Optional (use location of repository as cookbook path if nil)
+    attr_accessor :cookbooks_path
+    # (String) Private SSH key used to retrieve git repositories
+    # Optional, not used for svn and raw repositories.
+    attr_accessor :ssh_key
+    # (String) Username used to retrieve svn and raw repositories
+    # Optional, not used for git repositories.
+    attr_accessor :username
+    # (String) Password used to retrieve svn and raw repositories
+    # Optional, not used for git repositories.
+    attr_accessor :password
+    # (String) hash of the CookbookSequence that corresponds to the repo
+    attr_accessor :repo_sha
+    # (Array) List of cookbook <name, position> pairs
+    attr_accessor :positions
+
+    # Initialize fields from given arguments
+    def initialize(*args)
+      @repo_type              = args[0]
+      @url                    = args[1] if args.size > 1
+      @tag                    = args[2] if args.size > 2
+      @cookbooks_path         = args[3] if args.size > 3
+      @ssh_key                = args[4] if args.size > 4
+      @username               = args[5] if args.size > 5
+      @password               = args[6] if args.size > 6
+      @repo_sha               = args[7] if args.size > 7
+      @positions              = args[8] if args.size > 8
+    end
+
+    # Array of serialized fields given to constructor
+    def serialized_members
+      [ @repo_type, @url, @tag, @cookbooks_path, @ssh_key, @username, @password, @repo_sha, @positions ]
+    end
+  end
+end

data/lib/right_agent/log.rb

@@ -319,8 +319,9 @@ module RightScale
         if new_level != @level
           @logger.info("[setup] Setting log level to #{level_to_sym(new_level).to_s.upcase}")
           @logger.level = @level = new_level
-          @notify.each { |n| n.call(@level) } if @notify
         end
+        # Notify even if unchanged since don't know when callback was set
+        @notify.each { |n| n.call(@level) } if @notify
       end
       level = level_to_sym(@level)
     end

data/lib/right_agent/monkey_patches/amqp_patch.rb

@@ -20,48 +20,170 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-begin
-  # Clean up AMQP connection when an error is raised after a broker request failure,
-  # otherwise AMQP becomes unusable
-  AMQP.module_eval do
-    def self.start *args, &blk
-      begin
-        EM.run{
-          @conn ||= connect *args
-          @conn.callback { AMQP.channel = AMQP::Channel.new(@conn) }
+class MQ
 
-          # callback passed to .start must come last
-          @conn.callback(&blk) if blk
-          @conn
+  class Queue
+    # Asks the broker to redeliver all unacknowledged messages on a
+    # specified channel. Zero or more messages may be redelivered.
+    #
+    # * requeue (default false)
+    # If this parameter is false, the message will be redelivered to the original recipient.
+    # If this flag is true, the server will attempt to requeue the message, potentially then
+    # delivering it to an alternative subscriber.
+    #
+    def recover(requeue = false)
+      @mq.callback{
+        @mq.send Protocol::Basic::Recover.new({ :requeue => requeue })
+      }
+      self
+    end
+  end
+
+  # May raise a MQ::Error exception when the frame payload contains a
+  # Protocol::Channel::Close object.
+  #
+  # This usually occurs when a client attempts to perform an illegal
+  # operation. A short, and incomplete, list of potential illegal operations
+  # follows:
+  # * publish a message to a deleted exchange (NOT_FOUND)
+  # * declare an exchange using the reserved 'amq.' naming structure (ACCESS_REFUSED)
+  #
+  def process_frame frame
+    log :received, frame
+
+    case frame
+    when Frame::Header
+      @header = frame.payload
+      @body = ''
+
+    when Frame::Body
+      @body << frame.payload
+      if @body.length >= @header.size
+        if @method.is_a? Protocol::Basic::Return
+          @on_return_message.call @method, @body if @on_return_message
+        else
+          @header.properties.update(@method.arguments)
+          @consumer.receive @header, @body if @consumer
+        end
+        @body = @header = @consumer = @method = nil
+      end
+
+    when Frame::Method
+      case method = frame.payload
+      when Protocol::Channel::OpenOk
+        send Protocol::Access::Request.new(:realm => '/data',
+                                           :read => true,
+                                           :write => true,
+                                           :active => true,
+                                           :passive => true)
+
+      when Protocol::Access::RequestOk
+        @ticket = method.ticket
+        callback{
+          send Protocol::Channel::Close.new(:reply_code => 200,
+                                            :reply_text => 'bye',
+                                            :method_id => 0,
+                                            :class_id => 0)
+        } if @closing
+        succeed
+
+      when Protocol::Basic::CancelOk
+        if @consumer = consumers[ method.consumer_tag ]
+          @consumer.cancelled
+        else
+          MQ.error "Basic.CancelOk for invalid consumer tag: #{method.consumer_tag}"
+        end
+
+      when Protocol::Queue::DeclareOk
+        queues[ method.queue ].receive_status method
+
+      when Protocol::Basic::Deliver, Protocol::Basic::GetOk
+        @method = method
+        @header = nil
+        @body = ''
+
+        if method.is_a? Protocol::Basic::GetOk
+          @consumer = get_queue{|q| q.shift }
+          MQ.error "No pending Basic.GetOk requests" unless @consumer
+        else
+          @consumer = consumers[ method.consumer_tag ]
+          MQ.error "Basic.Deliver for invalid consumer tag: #{method.consumer_tag}" unless @consumer
+        end
+
+      when Protocol::Basic::GetEmpty
+        if @consumer = get_queue{|q| q.shift }
+          @consumer.receive nil, nil
+        else
+          MQ.error "Basic.GetEmpty for invalid consumer"
+        end
+
+      when Protocol::Basic::Return
+        @method = method
+        @header = nil
+        @body = ''
+
+      when Protocol::Channel::Close
+        raise Error, "#{method.reply_text} in #{Protocol.classes[method.class_id].methods[method.method_id]} on #{@channel}"
+
+      when Protocol::Channel::CloseOk
+        @closing = false
+        conn.callback{ |c|
+          c.channels.delete @channel
+          c.close if c.channels.empty?
         }
-      rescue Exception => e
-        @conn = nil
-        raise e
+
+      when Protocol::Basic::ConsumeOk
+        if @consumer = consumers[ method.consumer_tag ]
+          @consumer.confirm_subscribe
+        else
+          MQ.error "Basic.ConsumeOk for invalid consumer tag: #{method.consumer_tag}"
+        end
       end
     end
   end
 
+  # Provide callback to be activated when a message is returned
+  def return_message(&blk)
+    @on_return_message = blk
+  end
+
+end
+
+# monkey patch to the amqp gem that adds :no_declare => true option for new Queue objects.
+# This allows an instance that has no configuration privileges to enroll without blowing
+# up the AMQP gem when it tries to subscribe to its queue before it has been created.
+# Exchange :no_declare support is already in the eventmachine-0.12.10 gem.
+# temporary until we get this into amqp proper
+MQ::Queue.class_eval do
+  def initialize mq, name, opts = {}
+    @mq = mq
+    @opts = opts
+    @bindings ||= {}
+    @mq.queues[@name = name] ||= self
+    unless opts[:no_declare]
+      @mq.callback{
+        @mq.send AMQP::Protocol::Queue::Declare.new({ :queue => name,
+                                                      :nowait => true }.merge(opts))
+      }
+    end
+  end
+end
+
+begin
+  # Monkey patch AMQP reconnect backoff
   AMQP::Client.module_eval do
-    # Add callback for connection failure
     def initialize opts = {}
       @settings = opts
       extend AMQP.client
 
-      @_channel_mutex = Mutex.new
-
       @on_disconnect ||= proc{ @connection_status.call(:failed) if @connection_status }
 
       timeout @settings[:timeout] if @settings[:timeout]
       errback{ @on_disconnect.call } unless @reconnecting
-      @connection_status = @settings[:connection_status]
 
-      # TCP connection "openness"
-      @tcp_connection_established = false
-      # AMQP connection "openness"
-      @connected                  = false
+      @connected = false
     end
 
-    # Add backoff controls to the reconnect algorithm
     def reconnect(force = false)
       if @reconnecting and not force
         # Wait after first reconnect attempt and in between each subsequent attempt
@@ -94,96 +216,53 @@ begin
         "#{RightScale::AgentIdentity.new('rs', 'broker', @settings[:port].to_i, @settings[:host].gsub('-', '~')).to_s}")
       log 'reconnecting'
       EM.reconnect(@settings[:host], @settings[:port], self)
-    rescue Exception => e
-      RightScale::Log.error("Exception caught during AMQP reconnect", e, :trace)
-      reconnect if @reconnecting
     end
+  end
 
-    # Catch exceptions that would otherwise cause EM to stop or be in a bad state if a top
-    # level EM error handler was setup. Instead close the connection and leave EM alone.
-    # Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set
-    alias :orig_receive_data :receive_data
-    def receive_data(*args)
+  # Monkey patch AMQP to clean up @conn when an error is raised after a broker request failure,
+  # otherwise AMQP becomes unusable
+  AMQP.module_eval do
+    def self.start *args, &blk
       begin
-        orig_receive_data(*args)
+        EM.run{
+          @conn ||= connect *args
+          @conn.callback(&blk) if blk
+          @conn
+        }
       rescue Exception => e
-        unless ENV['IGNORE_AMQP_FAILURES']
-          RightScale::Log.error("Exception caught while processing AMQP frame, closing connection", e, :trace)
-        end
-        close_connection
+        @conn = nil
+        raise e
       end
     end
-
-    # Make it log to RightScale when logging enabled
-    def log(*args)
-      return unless @settings[:logging] or AMQP.logging
-      require 'pp'
-      RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
-    end
   end
 
-  AMQP::Channel.class_eval do
-    # Detect message return and make callback
-    def check_content_completion
-      if @body.length >= @header.size
-        if @method.is_a? AMQP::Protocol::Basic::Return
-          @on_return_message.call @method, @body if @on_return_message
-        else
-          @header.properties.update(@method.arguments)
-          @consumer.receive @header, @body if @consumer
-        end
-        @body = @header = @consumer = @method = nil
-      end
-    end
-
-    # Provide callback to be activated when a message is returned
-    def return_message(&blk)
-      @on_return_message = blk
-    end
-
-    # Apply :no_declare option
-    def validate_parameters_match!(entity, parameters)
-      unless entity.opts == parameters || parameters[:passive] || parameters[:no_declare] || entity.opts[:no_declare]
-        raise AMQP::IncompatibleOptionsError.new(entity.name, entity.opts, parameters)
+  # This monkey patch catches exceptions that would otherwise cause EM to stop or be in a bad
+  # state if a top level EM error handler was setup. Instead close the connection and leave EM
+  # alone.
+  # Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set (used in the 
+  # enroll script)
+  AMQP::Client.module_eval do
+    alias :orig_receive_data :receive_data
+    def receive_data(*args)
+      begin
+        orig_receive_data(*args)
+      rescue Exception => e
+        RightScale::Log.error("Exception caught while processing AMQP frame, closing connection",
+                                   e, :trace) unless ENV['IGNORE_AMQP_FAILURES']
+        close_connection
       end
     end
-
-    # Make it log to RightScale when logging enabled
-    def log(*args)
-      return unless AMQP.logging
-      require 'pp'
-      RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
-    end
   end
 
-  # Add :no_declare => true option for new Queue objects to allow an instance that has
-  # no configuration privileges to enroll without blowing up the AMQP gem when it tries
-  # to subscribe to its queue before it has been created (already supported in gem for
-  # Exchange)
-  AMQP::Queue.class_eval do
-    def initialize(mq, name, opts = {}, &block)
-      raise ArgumentError, "queue name must not be nil. Use '' (empty string) for server-named queues." if name.nil?
-
-      @mq = mq
-      @opts = self.class.add_default_options(name, opts, block)
-      @bindings ||= {}
-      @status = @opts[:nowait] ? :unknown : :unfinished
-
-      if name.empty?
-        @mq.queues_awaiting_declare_ok.push(self)
-      else
-        @name = name
-      end
-
-      unless opts[:no_declare]
-        @mq.callback{
-          @mq.send AMQP::Protocol::Queue::Declare.new(@opts)
-        }
-      end
-
-      self.callback = block
-
-      block.call(self) if @opts[:nowait] && block
+  # Add a new callback to amqp gem that triggers once the handshake with the broker completed
+  # The 'connected' status callback happens before the handshake is done and if it results in
+  # a lot of activity it might prevent EM from being able to call the code handling the
+  # incoming handshake packet in a timely fashion causing the broker to close the connection
+  AMQP::BasicClient.module_eval do
+    alias :orig_process_frame :process_frame
+    def process_frame(frame)
+      orig_process_frame(frame)
+      @connection_status.call(:ready) if @connection_status && frame.payload.is_a?(AMQP::Protocol::Connection::Start)
     end
   end